# allow crawling, but each page we want hidden has noindex,nofollow,noarchive so none of them should get indexed and only found if linked to externally 


User-agent: *
Allow: /
#