diff --git a/changelog.d/fediindex.change b/changelog.d/fediindex.change new file mode 100644 index 000000000..b9bef2762 --- /dev/null +++ b/changelog.d/fediindex.change @@ -0,0 +1 @@ +Allow FediIndex crawler bot by default \ No newline at end of file diff --git a/priv/static/robots.txt b/priv/static/robots.txt index 25781b7d7..4559309f4 100644 --- a/priv/static/robots.txt +++ b/priv/static/robots.txt @@ -1,2 +1,8 @@ -User-Agent: * -Disallow: +User-agent: * +Disallow: / + +User-agent: FediIndex +Allow: /nodeinfo/2.0 +Allow: /nodeinfo/2.1 +Allow: /nodeinfo/2.2 +Disallow: / \ No newline at end of file