User-agent: * Allow: / # The public, indexable surfaces. Allow: /bounty-board Allow: /talent Allow: /mentorship Allow: /courses Allow: /categories Allow: /bundles Allow: /store Allow: /monster-coin Allow: /legal # The Arabic half of every one of the above. Each is a separate # indexable URL with reciprocal hreflang tags, not a duplicate. Allow: /ar/ # Signed-in areas. None of these render anything a search result # should land on, and several are per-user , crawling them only # burns crawl budget. Disallow: /admin Disallow: /workspace Disallow: /ar/workspace Disallow: /student Disallow: /ar/student Disallow: /dashboard Disallow: /settings Disallow: /mentor/ # Auth and transactional flows. Disallow: /login Disallow: /register Disallow: /sign_in Disallow: /sign_up Disallow: /sign-in Disallow: /sign-up Disallow: /sign_out Disallow: /forgot-password Disallow: /reset-password Disallow: /email/verify Disallow: /checkout Disallow: /payment/ Disallow: /support/checkout # Server-rendered share cards , referenced by og:image (which # crawlers fetch regardless of this file) but images, not pages. Disallow: /*/share-image.png$ # Deliberately NOT disallowed: the directories' filtered, sorted # and paginated views (?sort=, ?skills=, ?page=). Disallow blocks # CRAWLING, not indexing, so blocking page 2 of the board would # stop Google ever reaching the bounties that only appear on it. # Those views send "noindex,follow" plus a canonical to the clean # URL instead , out of the index, still walkable to every listing. Sitemap: https://monstersgate.com/sitemap.xml