From 8a23358fc82f09853a3721d5f36ae4adac32b84b Mon Sep 17 00:00:00 2001
From: Nick Sweeting <git@nicksweeting.com>
Date: Wed, 12 Sep 2018 19:25:48 -0400
Subject: [PATCH] create robots.txt in output dir
---
archiver/index.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/archiver/index.py b/archiver/index.py
index 8c85f5c6..21f68697 100644
--- a/archiver/index.py
+++ b/archiver/index.py
@@ -72,6 +72,9 @@ def write_html_links_index(out_dir, links):
copy_tree(os.path.join(TEMPLATES_DIR, 'static'), os.path.join(out_dir, 'static'))
+ with open(os.path.join(out_dir, 'robots.txt'), 'w+') as f:
+ f.write('User-agent: *\nDisallow: /')
+
with open(os.path.join(TEMPLATES_DIR, 'index.html'), 'r', encoding='utf-8') as f:
index_html = f.read()