Algolia

{
  "plugins": [
    ["your-algolia-plugin-package", {
      "appId": "YOUR_APP_ID",
      "apiKey": "YOUR_SEARCH_ONLY_API_KEY",
      "indexName": "YOUR_INDEX_NAME",
      "assistantId": "optional_assistant_id"
    }]
  ]
}

new Crawler({
  appId: "YOUR_APP_ID",           // ← your App ID
  apiKey: "YOUR_CRAWLER_API_KEY",  // ← your crawler API key
  indexPrefix: "",
  rateLimit: 8,
  startUrls: ["https://your-domain.com"], // ← your domain
  renderJavaScript: true,
  maxDepth: 10,
  maxUrls: 8000,
  schedule: "every 1 day at 02:00 am",
  sitemaps: [],
  ignoreCanonicalTo: true,
  discoveryPatterns: ["https://your-domain.com/**"], // ← your domain
  actions: [
    {
      indexName: "YOUR_INDEX_NAME", // ← your index name
      pathsToMatch: ["https://your-domain.com/**"], // ← your domain
      recordExtractor: ({ $, helpers }) => {
        const lvl0 =
          // lvl0 = section name from breadcrumb (second item: Docs > Section > Page)
          $(".breadcrumbs li:nth-child(2)").text().trim() ||
          "Docs";

        const layoutAnchors = ["scroll-container", "main-navbar"];

        return helpers
          .docsearch({
            recordProps: {
              lvl0: {
                selectors: "",
                defaultValue: lvl0,
              },
              lvl1: [".prose h1"],
              content: [".prose p, .prose li"],
              lvl2: [".prose h2"],
              lvl3: [".prose h3"],
              lvl4: [".prose h4"],
              lvl5: [".prose h5"],
              lvl6: [".prose h6"],
            },
            aggregateContent: true,
            recordVersion: "v3",
          })
          .map((record) => {
            if (!layoutAnchors.includes(record.anchor)) return record;
            return {
              ...record,
              anchor: undefined,
              url: record.url
                ? record.url.replace(/#(scroll-container|main-navbar)$/, "")
                : record.url,
              url_without_anchor:
                record.url_without_anchor ||
                (record.url ? record.url.split("#")[0] : record.url),
            };
          });
      },
    },
  ],
  safetyChecks: { beforeIndexPublishing: { maxLostRecordsPercentage: 10 } },
  initialIndexSettings: {
    "YOUR_INDEX_NAME": {
      attributesForFaceting: ["type", "lang"],
      attributesToRetrieve: [
        "hierarchy", "content", "anchor", "url",
        "url_without_anchor", "type",
      ],
      attributesToHighlight: ["hierarchy", "content"],
      attributesToSnippet: ["content:10"],
      camelCaseAttributes: ["hierarchy", "content"],
      searchableAttributes: [
        "unordered(hierarchy.lvl0)",
        "unordered(hierarchy.lvl1)",
        "unordered(hierarchy.lvl2)",
        "unordered(hierarchy.lvl3)",
        "unordered(hierarchy.lvl4)",
        "unordered(hierarchy.lvl5)",
        "unordered(hierarchy.lvl6)",
        "content",
      ],
      distinct: true,
      attributeForDistinct: "url",
      customRanking: [
        "desc(weight.pageRank)",
        "desc(weight.level)",
        "asc(weight.position)",
      ],
      ranking: [
        "words", "filters", "typo", "attribute",
        "proximity", "exact", "custom",
      ],
      highlightPreTag: '<span class="algolia-docsearch-suggestion--highlight">',
      highlightPostTag: "</span>",
      minWordSizefor1Typo: 3,
      minWordSizefor2Typos: 7,
      allowTyposOnNumericTokens: false,
      minProximity: 1,
      ignorePlurals: true,
      advancedSyntax: true,
      attributeCriteriaComputedByMinProximity: true,
      removeWordsIfNoResults: "allOptional",
    },
  },
});