# ==============================================================================
# ALBUQUERQUE JOURNAL COPYRIGHT & AUTOMATED CRAWLING NOTICE
# Copyright (c) Albuquerque Journal. All rights reserved.
# 
# Use of any automated device, crawler, scraper, or mining tool to collect 
# content from abqjournal.com is strictly restricted. This includes:
# (1) Text and data mining (TDM) activities;
# (2) Development, training, or ingestion into Artificial Intelligence (AI) models, 
#     Machine Learning (ML) tools, or Large Language Models (LLMs);
# (3) Archiving, caching, or creating derivative databases of our reporting;
# (4) Any unauthorized commercial reproduction or redistribution.
# ==============================================================================

# ==============================================================================
# SECTION 1: DEFAULT CRAWLER RULES (Search Engines, Local Geo & Sponsored)
# Allows standard news, local/geo routes, and commercial content while 
# protecting backend admin paths and dynamic utilities.
# ==============================================================================
User-agent: *
# Standard News & Editorial Sections (Allowed)
Allow: /
Allow: /news/
Allow: /sports/
Allow: /business/
Allow: /lifestyles/
Allow: /obits/
Allow: /opinion/
Allow: /legals/
Allow: /events/

# Geo-Targeted, Local, & Regional Content Sections (Allowed)
Allow: /geo/
Allow: /geo*
Allow: /location/
Allow: /location*
Allow: /local/
Allow: /albuquerque/
Allow: /santa-fe/
Allow: /rio-rancho/
Allow: /new-mexico/
Allow: /weather/

# Branded, Sponsored & Commercial Partner Content (Allowed for GEO)
Allow: /sponsored/
Allow: /sponsored*
Allow: /branded/
Allow: /branded*
Allow: /native/
Allow: /partner/
Allow: /special-sections/

# Disallow Internal Search & Dynamic Query Parameters
Disallow: /search/
Disallow: /search*
Disallow: /*?*q=
Disallow: /*?*query=
Disallow: /*?*s=
Disallow: /*?*outputType=
Disallow: /*?*utm_source=

# Disallow User Accounts, Paywall, & Subscription Flows
Disallow: /login/
Disallow: /login*
Disallow: /account/
Disallow: /account/*
Disallow: /checkout/
Disallow: /checkout*
Disallow: /register/
Disallow: /register*
Disallow: /subscribe/
Disallow: /subscribe*
Disallow: /forgot-password/

# Disallow Labrador CMS Backend, Private APIs, & Internal Trackers
Disallow: /admin/
Disallow: /labrador/
Disallow: /cdn-cgi/
Disallow: /partials/
Disallow: /track-analytics/
Disallow: /tracking/
Disallow: /ads/
Disallow: /adx/
Disallow: /ad/

# Allow Public Geolocation APIs & Services
Allow: /api/geo/
Allow: /api/location/

# Disallow Unpopulated/Generic Author Index Routes
Disallow: /author/
Disallow: /author$

# ==============================================================================
# SECTION 2: AI MODEL TRAINING SCRAPERS (HARDBLOCK)
# Prohibits automated data harvesting used to train base Large Language Models (LLMs).
# ==============================================================================
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: CCBot
User-agent: Bytespider
User-agent: Applebot-Extended
User-agent: cohere-training-data-crawler
User-agent: Diffbot
User-agent: Meta-ExternalAgent
User-agent: meta-externalagent
User-agent: Meta-ExternalFetcher
User-agent: meta-externalfetcher
User-agent: YouBot
User-agent: ImagesiftBot
User-agent: Amazonbot
Disallow: /

# ==============================================================================
# SECTION 3: REAL-TIME AI SEARCH & CITATION BOTS (EXPLICITLY ALLOWED FOR GEO/AEO)
# Permits live retrieval bots so AI search engines can cite the Journal & partners.
# ==============================================================================
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Claude-SearchBot
User-agent: GoogleOther
User-agent: Google-Extended
Allow: /
Allow: /business/
Allow: /sponsored/
Allow: /sponsored*
Allow: /branded/
Allow: /branded*
Allow: /local/

# ==============================================================================
# SECTION 4: PRIORITY SEARCH ENGINES & NEWS AGGREGATORS
# ==============================================================================
User-agent: Googlebot
User-agent: Googlebot-News
User-agent: Bingbot
User-agent: DuckDuckBot
Allow: /
Allow: /geo/
Allow: /local/
Allow: /sponsored/
Allow: /branded/
Allow: /special-sections/

# ==============================================================================
# SECTION 5: XML SITEMAPS
# ==============================================================================
Sitemap: https://www.abqjournal.com/sitemapindex.xml
Sitemap: https://www.abqjournal.com/news?lab_viewport=newssitemap