mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-13 06:34:25 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge pull request #357 from cdvv7788/mock-poc

Browse source
This commit is contained in:
Nick Sweeting 2020-07-13 11:27:35 -04:00 • committed by GitHub
commit a3eb51fa42
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 487 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

1
setup.py
View file

@ -65,6 +65,7 @@ setuptools.setup(
"sphinx-rtd-theme", "sphinx-rtd-theme",
"recommonmark", "recommonmark",
"pytest", "pytest",
"bottle",
], ],
# 'redis': ['redis', 'django-redis'], # 'redis': ['redis', 'django-redis'],
# 'pywb': ['pywb', 'redis'], # 'pywb': ['pywb', 'redis'],

19
tests/conftest.py Normal file
View file

@ -0,0 +1,19 @@
from multiprocessing import Process
import pytest
from .mock_server.server import start
server_process = None
@pytest.hookimpl
def pytest_sessionstart(session):
global server_process
server_process = Process(target=start)
server_process.start()
@pytest.hookimpl
def pytest_sessionfinish(session):
if server_process is not None:
server_process.terminate()
server_process.join()

0
tests/mock_server/__init__.py Normal file
View file

17
tests/mock_server/server.py Normal file
View file

@ -0,0 +1,17 @@
from os.path import abspath
from os import getcwd
from pathlib import Path
from bottle import route, run, static_file
@route("/")
def index():
return "Hello"
@route("/static/<filename>")
def static_path(filename):
template_path = abspath(getcwd()) / Path("tests/mock_server/templates")
return static_file(filename, root=template_path)
def start():
run(host='localhost', port=8080)

49
tests/mock_server/templates/example.com.html Normal file
View file

@ -0,0 +1,49 @@
<!doctype html>
<html>
<head>
<title>Example Domain</title>
<meta charset="utf-8"/>
<meta http-equiv="Content-type" content="text/html; charset=utf-8"/>
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<style type="text/css">
body {
background-color: #f0f0f2;
margin: 0;
padding: 0;
font-family: -apple-system, system-ui, BlinkMacSystemFont, "Segoe UI", "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
}
div {
width: 600px;
margin: 5em auto;
padding: 2em;
background-color: #fdfdff;
border-radius: 0.5em;
box-shadow: 2px 3px 7px 2px rgba(0, 0, 0, 0.02);
}
a:link,
a:visited {
color: #38488f;
text-decoration: none;
}
@media(max-width: 700px) {
div {
margin: 0 auto;
width: auto;
}
}
</style>
</head>
<body>
<div>
<h1>Example Domain</h1>
<p>This domain is for use in illustrative examples in documents. You may use this
domain in literature without prior coordination or asking for permission.</p>
<p>
<a href="http://localhost:8080/static/iana.org.html">More information...</a>
</p>
</div>
</body>
</html>

390
tests/mock_server/templates/iana.org.html Normal file
View file

@ -0,0 +1,390 @@
<!doctype html>
<html>
<head>
<title>IANA — IANA-managed Reserved Domains</title>
<meta charset="utf-8"/>
<meta http-equiv="Content-type" content="text/html; charset=utf-8"/>
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<link rel="stylesheet" media="screen" href="/_css/2015.1/screen.css"/>
<link rel="stylesheet" media="print" href="/_css/2015.1/print.css"/>
<link rel="shortcut icon" type="image/ico" href="/_img/bookmark_icon.ico"/>
<script type="text/javascript" src="/_js/2013.1/jquery.js"></script>
<script type="text/javascript" src="/_js/2013.1/iana.js"></script>
</head>
<body>
<header>
<div id="header">
<div id="logo">
<a href="/"><img src="/_img/2013.1/iana-logo-header.svg" alt="Homepage"/></a>
</div>
<div class="navigation">
<ul>
<li><a href="/domains">Domains</a></li>
<li><a href="/numbers">Numbers</a></li>
<li><a href="/protocols">Protocols</a></li>
<li><a href="/about">About Us</a></li>
</ul>
</div>
</div>
</header>
<div id="body">
<div id="main_right">
<h1>IANA-managed Reserved Domains</h1>
<p>Certain domains are set aside, and nominally registered to &ldquo;IANA&rdquo;, for specific
policy or technical purposes.</p>
<h2>Example domains</h2>
<p>As described in
<a href="/go/rfc2606">RFC 2606</a>
and
<a href="/go/rfc6761">RFC 6761</a>,
a number of domains such as
<span class="domain label">example.com</span>
and
<span class="domain label">example.org</span>
are maintained for documentation purposes. These domains may be used as illustrative
examples in documents without prior coordination with us. They are
not available for registration or transfer.</p>
<h2>Test IDN top-level domains</h2>
<p>These domains were temporarily delegated by IANA for the
<a href="http://www.icann.org/topics/idn/">IDN Evaluation</a>
being conducted by
<a href="http://www.icann.org/">ICANN</a>.</p>
<div class="iana-table-frame">
<table id="arpa-table" class="iana-table">
<thead>
<tr>
<th>Domain</th>
<th>Domain (A-label)</th>
<th>Language</th>
<th>Script</th>
</tr>
</thead>
<tbody>
<tr>
<td>&#1573;&#1582;&#1578;&#1576;&#1575;&#1585;</td>
<td>
<span class="domain label">
<a href="/domains/root/db/xn--kgbechtv.html">XN--KGBECHTV</a>
</span>
</td>
<td>Arabic</td>
<td>Arabic</td>
</tr>
<tr>
<td>&#1570;&#1586;&#1605;&#1575;&#1740;&#1588;&#1740;</td>
<td>
<span class="domain label">
<a href="/domains/root/db/xn--hgbk6aj7f53bba.html">XN--HGBK6AJ7F53BBA</a>
</span>
</td>
<td>Persian</td>
<td>Arabic</td>
</tr>
<tr>
<td>&#27979;&#35797;</td>
<td>
<span class="domain label">
<a href="/domains/root/db/xn--0zwm56d.html">XN--0ZWM56D</a>
</span>
</td>
<td>Chinese</td>
<td>Han (Simplified variant)</td>
</tr>
<tr>
<td>&#28204;&#35430;</td>
<td>
<span class="domain label">
<a href="/domains/root/db/xn--g6w251d.html">XN--G6W251D</a>
</span>
</td>
<td>Chinese</td>
<td>Han (Traditional variant)</td>
</tr>
<tr>
<td>&#1080;&#1089;&#1087;&#1099;&#1090;&#1072;&#1085;&#1080;&#1077;</td>
<td>
<span class="domain label">
<a href="/domains/root/db/xn--80akhbyknj4f.html">XN--80AKHBYKNJ4F</a>
</span>
</td>
<td>Russian</td>
<td>Cyrillic</td>
</tr>
<tr>
<td>&#2346;&#2352;&#2368;&#2325;&#2381;&#2359;&#2366;</td>
<td>
<span class="domain label">
<a href="/domains/root/db/xn--11b5bs3a9aj6g.html">XN--11B5BS3A9AJ6G</a>
</span>
</td>
<td>Hindi</td>
<td>Devanagari (Nagari)</td>
</tr>
<tr>
<td>&#948;&#959;&#954;&#953;&#956;&#942;</td>
<td>
<span class="domain label">
<a href="/domains/root/db/xn--jxalpdlp.html">XN--JXALPDLP</a>
</span>
</td>
<td>Greek, Modern (1453-)</td>
<td>Greek</td>
</tr>
<tr>
<td>&#53580;&#49828;&#53944;</td>
<td>
<span class="domain label">
<a href="/domains/root/db/xn--9t4b11yi5a.html">XN--9T4B11YI5A</a>
</span>
</td>
<td>Korean</td>
<td>Hangul (Hang&#x16D;l, Hangeul)</td>
</tr>
<tr>
<td>&#1496;&#1506;&#1505;&#1496;</td>
<td>
<span class="domain label">
<a href="/domains/root/db/xn--deba0ad.html">XN--DEBA0AD</a>
</span>
</td>
<td>Yiddish</td>
<td>Hebrew</td>
</tr>
<tr>
<td>&#12486;&#12473;&#12488;</td>
<td>
<span class="domain label">
<a href="/domains/root/db/xn--zckzah.html">XN--ZCKZAH</a>
</span>
</td>
<td>Japanese</td>
<td>Katakana</td>
</tr>
<tr>
<td>&#2986;&#2992;&#3007;&#2975;&#3021;&#2970;&#3016;</td>
<td>
<span class="domain label">
<a href="/domains/root/db/xn--hlcj6aya9esc7a.html">XN--HLCJ6AYA9ESC7A</a>
</span>
</td>
<td>Tamil</td>
<td>Tamil</td>
</tr>
</tbody>
</table>
</div>
<h2>Policy-reserved domains</h2>
<p>We act as both the registrant and registrar for a select number of domains
which have been reserved under policy grounds. These exclusions are
typically indicated in either technical standards (RFC documents),
or
<a href="http://www.icann.org/en/registries/agreements.htm">contractual limitations</a>.</p>
<p>Domains which are described as registered to IANA or ICANN on policy
grounds are not available for registration or transfer, with the exception
of
<span class="domain label">
<i>country-name</i>.info</span>
domains. These domains are available for release
by the ICANN Governmental Advisory Committee Secretariat.</p>
<h2>Other Special-Use Domains</h2>
<p>There is additionally a
<a href="/assignments/special-use-domain-names">Special-Use Domain Names</a>
registry documenting special-use domains designated by technical standards. For further information, see
<a href="/go/rfc6761">Special-Use Domain Names</a>
(RFC 6761).</p>
</div>
<div id="sidebar_left">
<div class="navigation_box">
<h2>Domain Names</h2>
<ul>
<li id="nav_dom_top">
<a href="/domains">Overview</a>
</li>
<li id="nav_dom_root">
<a href="/domains/root">Root Zone Management</a>
</li>
<ul id="nav_dom_root_sub">
<li id="nav_dom_root_top">
<a href="/domains/root">Overview</a>
</li>
<li id="nav_dom_root_db">
<a href="/domains/root/db">Root Database</a>
</li>
<li id="nav_dom_root_files">
<a href="/domains/root/files">Hint and Zone Files</a>
</li>
<li id="nav_dom_root_manage">
<a href="/domains/root/manage">Change Requests</a>
</li>
<li id="nav_dom_root_procedures">
<a href="/domains/root/help">Instructions &amp; Guides</a>
</li>
<li id="nav_dom_root_servers">
<a href="/domains/root/servers">Root Servers</a>
</li>
</ul>
<li id="nav_dom_int">
<a href="/domains/int">.INT Registry</a>
</li>
<ul id="nav_dom_int_sub">
<li id="nav_dom_int_top">
<a href="/domains/int">Overview</a>
</li>
<li id="nav_dom_int_manage">
<a href="/domains/int/manage">Register/modify an .INT domain</a>
</li>
<li id="nav_dom_int_policy">
<a href="/domains/int/policy">Eligibility</a>
</li>
</ul>
<li id="nav_dom_arpa">
<a href="/domains/arpa">.ARPA Registry</a>
</li>
<li id="nav_dom_idn">
<a href="/domains/idn-tables">IDN Practices Repository</a>
</li>
<ul id="nav_dom_idn_sub">
<li id="nav_dom_idn_top">
<a href="/domains/idn-tables">Overview</a>
</li>
<!-- <li id="nav_dom_idn_tables"><a href="/domains/idn-tables/db">Tables</a></li> -->
<li id="nav_dom_idn_submit">
<a href="/procedures/idn-repository.html">Submit a table</a>
</li>
</ul>
<li id="nav_dom_dnssec">
<a href="/dnssec">Root Key Signing Key (DNSSEC)</a>
</li>
<ul id="nav_dom_dnssec_sub">
<li id="nav_dom_dnssec_top">
<a href="/dnssec">Overview</a>
</li>
<li id="nav_dom_dnssec_ksk">
<a href="/dnssec/files">Trusts Anchors and Keys</a>
</li>
<li id="nav_dom_dnssec_ceremonies">
<a href="/dnssec/ceremonies">Root KSK Ceremonies</a>
</li>
<li id="nav_dom_dnssec_dps">
<a href="/dnssec/dps">Practice Statement</a>
</li>
<li id="nav_dom_dnssec_tcrs">
<a href="/dnssec/tcrs">Community Representatives</a>
</li>
</ul>
<li id="nav_dom_special">
<a href="/domains/reserved">Reserved Domains</a>
</li>
</ul>
</div>
</div>
</div>
<footer>
<div id="footer">
<table class="navigation">
<tr>
<td class="section">
<a href="/domains">Domain&nbsp;Names</a>
</td>
<td class="subsection">
<ul>
<li><a href="/domains/root">Root Zone Registry</a></li>
<li><a href="/domains/int">.INT Registry</a></li>
<li><a href="/domains/arpa">.ARPA Registry</a></li>
<li><a href="/domains/idn-tables">IDN Repository</a></li>
</ul>
</td>
</tr>
<tr>
<td class="section">
<a href="/numbers">Number&nbsp;Resources</a>
</td>
<td class="subsection">
<ul>
<li><a href="/abuse">Abuse Information</a></li>
</ul>
</td>
</tr>
<tr>
<td class="section">
<a href="/protocols">Protocols</a>
</td>
<td class="subsection">
<ul>
<li><a href="/protocols">Protocol Registries</a></li>
<li><a href="/time-zones">Time Zone Database</a></li>
</ul>
</td>
</tr>
<tr>
<td class="section">
<a href="/about">About&nbsp;Us</a>
</td>
<td class="subsection">
<ul>
<li><a href="/about/presentations">Presentations</a></li>
<li><a href="/reports">Reports</a></li>
<li><a href="/performance">Performance</a></li>
<li><a href="/reviews">Reviews</a></li>
<li><a href="/about/excellence">Excellence</a></li>
<li><a href="/contact">Contact Us</a></li>
</ul>
</td>
</tr>
</table>
<div id="custodian">
<p>The IANA functions coordinate the Internet’s globally unique identifiers, and
are provided by
<a href="http://pti.icann.org">Public Technical Identifiers</a>, an affiliate of
<a href="http://www.icann.org/">ICANN</a>.</p>
</div>
<div id="legalnotice">
<ul>
<li><a href="https://www.icann.org/privacy/policy">Privacy Policy</a></li>
<li><a href="https://www.icann.org/privacy/tos">Terms of Service</a></li>
</ul>
</p>
</div>
</div>
</body>
</html></footer><script>
$(document).ready(function () {
$("#nav_dom_special").addClass("selected")
$("#nav_dom_int_sub").hide()
$("#nav_dom_idn_sub").hide()
$("#nav_dom_dnssec_sub").hide()
$("#nav_dom_tools_sub").hide()
$("#nav_dom_root_sub").hide()
});</script></body></html>

16
tests/test_args.py
View file

@ -4,25 +4,25 @@ import json
from .fixtures import * from .fixtures import *
def test_depth_flag_is_accepted(process): def test_depth_flag_is_accepted(process):
arg_process = subprocess.run(["archivebox", "add", "https://example.com", "--depth=0"], capture_output=True) arg_process = subprocess.run(["archivebox", "add", "http://localhost:8080/static/example.com.html", "--depth=0"], capture_output=True)
assert 'unrecognized arguments: --depth' not in arg_process.stderr.decode("utf-8") assert 'unrecognized arguments: --depth' not in arg_process.stderr.decode("utf-8")
def test_depth_flag_fails_if_it_is_not_0_or_1(process): def test_depth_flag_fails_if_it_is_not_0_or_1(process):
arg_process = subprocess.run(["archivebox", "add", "https://example.com", "--depth=5"], capture_output=True) arg_process = subprocess.run(["archivebox", "add", "http://localhost:8080/static/example.com.html", "--depth=5"], capture_output=True)
assert 'invalid choice' in arg_process.stderr.decode("utf-8") assert 'invalid choice' in arg_process.stderr.decode("utf-8")
arg_process = subprocess.run(["archivebox", "add", "https://example.com", "--depth=-1"], capture_output=True) arg_process = subprocess.run(["archivebox", "add", "http://localhost:8080/static/example.com.html", "--depth=-1"], capture_output=True)
assert 'invalid choice' in arg_process.stderr.decode("utf-8") assert 'invalid choice' in arg_process.stderr.decode("utf-8")
def test_depth_flag_0_crawls_only_the_arg_page(tmp_path, process): def test_depth_flag_0_crawls_only_the_arg_page(tmp_path, process):
arg_process = subprocess.run(["archivebox", "add", "https://example.com", "--depth=0"], capture_output=True) arg_process = subprocess.run(["archivebox", "add", "http://localhost:8080/static/example.com.html", "--depth=0"], capture_output=True)
archived_item_path = list(tmp_path.glob('archive/**/*'))[0] archived_item_path = list(tmp_path.glob('archive/**/*'))[0]
with open(archived_item_path / "index.json", "r") as f: with open(archived_item_path / "index.json", "r") as f:
output_json = json.load(f) output_json = json.load(f)
assert output_json["base_url"] == "example.com" assert output_json["base_url"] == "localhost:8080/static/example.com.html"
def test_depth_flag_1_crawls_the_page_AND_links(tmp_path, process): def test_depth_flag_1_crawls_the_page_AND_links(tmp_path, process):
arg_process = subprocess.run(["archivebox", "add", "https://example.com", "--depth=1"], capture_output=True) arg_process = subprocess.run(["archivebox", "add", "http://localhost:8080/static/example.com.html", "--depth=1"], capture_output=True)
with open(tmp_path / "index.json", "r") as f: with open(tmp_path / "index.json", "r") as f:
archive_file = f.read() archive_file = f.read()
assert "https://example.com" in archive_file assert "http://localhost:8080/static/example.com.html" in archive_file
assert "https://www.iana.org/domains/example" in archive_file assert "http://localhost:8080/static/iana.org.html" in archive_file

4
tests/test_init.py
View file

@ -18,7 +18,7 @@ def test_update(tmp_path, process):
def test_add_link(tmp_path, process): def test_add_link(tmp_path, process):
os.chdir(tmp_path) os.chdir(tmp_path)
add_process = subprocess.run(['archivebox', 'add', 'http://example.com'], capture_output=True) add_process = subprocess.run(['archivebox', 'add', 'http://localhost:8080/static/example.com.html'], capture_output=True)
archived_item_path = list(tmp_path.glob('archive/**/*'))[0] archived_item_path = list(tmp_path.glob('archive/**/*'))[0]
assert "index.json" in [x.name for x in archived_item_path.iterdir()] assert "index.json" in [x.name for x in archived_item_path.iterdir()]
@ -34,7 +34,7 @@ def test_add_link(tmp_path, process):
def test_add_link_support_stdin(tmp_path, process): def test_add_link_support_stdin(tmp_path, process):
os.chdir(tmp_path) os.chdir(tmp_path)
stdin_process = subprocess.Popen(["archivebox", "add"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdin_process = subprocess.Popen(["archivebox", "add"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
stdin_process.communicate(input="http://example.com".encode()) stdin_process.communicate(input="http://localhost:8080/static/example.com.html".encode())
archived_item_path = list(tmp_path.glob('archive/**/*'))[0] archived_item_path = list(tmp_path.glob('archive/**/*'))[0]
assert "index.json" in [x.name for x in archived_item_path.iterdir()] assert "index.json" in [x.name for x in archived_item_path.iterdir()]

2
tests/test_util.py
View file

@ -1,5 +1,5 @@
from archivebox import util from archivebox import util
def test_download_url_downloads_content(): def test_download_url_downloads_content():
text = util.download_url("https://example.com") text = util.download_url("http://localhost:8080/static/example.com.html")
assert "Example Domain" in text assert "Example Domain" in text