diff --git a/archivebox/core/admin.py b/archivebox/core/admin.py
index 52e4fb5a..d201e878 100644
--- a/archivebox/core/admin.py
+++ b/archivebox/core/admin.py
@@ -29,6 +29,7 @@ from core.mixins import SearchResultsAdminMixin
from api.models import APIToken
from abid_utils.admin import ABIDModelAdmin
from queues.tasks import bg_archive_links, bg_add
+from machine.models import Machine, NetworkInterface
from index.html import snapshot_icons
from logging_util import printable_filesize
@@ -778,3 +779,53 @@ class CustomWebhookAdmin(WebhookAdmin, ABIDModelAdmin):
list_display = ('created_at', 'created_by', 'abid', *WebhookAdmin.list_display)
sort_fields = ('created_at', 'created_by', 'abid', 'referenced_model', 'endpoint', 'last_success', 'last_error')
readonly_fields = ('created_at', 'modified_at', 'abid_info', *WebhookAdmin.readonly_fields)
+
+
+@admin.register(Machine, site=archivebox_admin)
+class MachineAdmin(ABIDModelAdmin):
+ list_display = ('abid', 'created_at', 'hostname', 'ips', 'os_platform', 'hw_in_docker', 'hw_in_vm', 'hw_manufacturer', 'hw_product', 'os_arch', 'os_family', 'os_release', 'hw_uuid')
+ sort_fields = ('abid', 'created_at', 'hostname', 'ips', 'os_platform', 'hw_in_docker', 'hw_in_vm', 'hw_manufacturer', 'hw_product', 'os_arch', 'os_family', 'os_release', 'hw_uuid')
+ # search_fields = ('id', 'abid', 'guid', 'hostname', 'hw_manufacturer', 'hw_product', 'hw_uuid', 'os_arch', 'os_family', 'os_platform', 'os_kernel', 'os_release')
+
+ readonly_fields = ('guid', 'created_at', 'modified_at', 'abid_info', 'ips')
+ fields = (*readonly_fields, 'hostname', 'hw_in_docker', 'hw_in_vm', 'hw_manufacturer', 'hw_product', 'hw_uuid', 'os_arch', 'os_family', 'os_platform', 'os_kernel', 'os_release', 'stats')
+
+ list_filter = ('hw_in_docker', 'hw_in_vm', 'os_arch', 'os_family', 'os_platform')
+ ordering = ['-created_at']
+ list_per_page = 100
+
+ @admin.display(
+ description='Public IP',
+ ordering='networkinterface__ip_public',
+ )
+ def ips(self, machine):
+ return format_html(
+ '{}',
+ machine.abid,
+ ', '.join(machine.networkinterface_set.values_list('ip_public', flat=True)),
+ )
+
+@admin.register(NetworkInterface, site=archivebox_admin)
+class NetworkInterfaceAdmin(ABIDModelAdmin):
+ list_display = ('abid', 'created_at', 'machine_info', 'ip_public', 'dns_server', 'isp', 'country', 'region', 'city', 'iface', 'ip_local', 'mac_address')
+ sort_fields = ('abid', 'created_at', 'machine_info', 'ip_public', 'dns_server', 'isp', 'country', 'region', 'city', 'iface', 'ip_local', 'mac_address')
+ search_fields = ('abid', 'machine__abid', 'iface', 'ip_public', 'ip_local', 'mac_address', 'dns_server', 'hostname', 'isp', 'city', 'region', 'country')
+
+ readonly_fields = ('machine', 'created_at', 'modified_at', 'abid_info', 'mac_address', 'ip_public', 'ip_local', 'dns_server')
+ fields = (*readonly_fields, 'iface', 'hostname', 'isp', 'city', 'region', 'country')
+
+ list_filter = ('isp', 'country', 'region')
+ ordering = ['-created_at']
+ list_per_page = 100
+
+ @admin.display(
+ description='Machine',
+ ordering='machine__abid',
+ )
+ def machine_info(self, iface):
+ return format_html(
+ '[{}] {}',
+ iface.machine.id,
+ iface.machine.abid,
+ iface.machine.hostname,
+ )
diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py
index c76979e1..424b0acb 100644
--- a/archivebox/core/settings.py
+++ b/archivebox/core/settings.py
@@ -98,7 +98,8 @@ INSTALLED_APPS = [
'django_object_actions', # provides easy Django Admin action buttons on change views https://github.com/crccheck/django-object-actions
# Our ArchiveBox-provided apps
- #'config', # ArchiveBox config settings
+ #'config', # ArchiveBox config settings (loaded as a plugin, don't need to add it here)
+ 'machine', # handles collecting and storing information about the host machine, network interfaces, installed binaries, etc.
'queues', # handles starting and managing background workers and processes
'abid_utils', # handles ABID ID creation, handling, and models
'core', # core django model with Snapshot, ArchiveResult, etc.
diff --git a/archivebox/machine/apps.py b/archivebox/machine/apps.py
new file mode 100644
index 00000000..f5c0867b
--- /dev/null
+++ b/archivebox/machine/apps.py
@@ -0,0 +1,8 @@
+from django.apps import AppConfig
+
+
+class MachineConfig(AppConfig):
+ default_auto_field = 'django.db.models.BigAutoField'
+
+ name = 'machine'
+ verbose_name = 'Machine Info'
diff --git a/archivebox/machine/detect.py b/archivebox/machine/detect.py
new file mode 100644
index 00000000..4a8a838a
--- /dev/null
+++ b/archivebox/machine/detect.py
@@ -0,0 +1,317 @@
+import os
+import json
+import socket
+import urllib.request
+from typing import Dict, Any
+from pathlib import Path
+import subprocess
+import platform
+import tempfile
+from datetime import datetime
+
+import psutil
+import machineid # https://github.com/keygen-sh/py-machineid
+
+from rich import print
+
+PACKAGE_DIR = Path(__file__).parent
+DATA_DIR = Path('.').resolve()
+
+def get_vm_info():
+ hw_in_docker = bool(os.getenv('IN_DOCKER', False) in ('1', 'true', 'True', 'TRUE'))
+ hw_in_vm = False
+ try:
+ # check for traces of docker/containerd/podman in cgroup
+ with open('/proc/self/cgroup', 'r') as procfile:
+ for line in procfile:
+ cgroup = line.strip() # .split('/', 1)[-1].lower()
+ if 'docker' in cgroup or 'containerd' in cgroup or 'podman' in cgroup:
+ hw_in_docker = True
+ except Exception:
+ pass
+
+ hw_manufacturer = 'Docker' if hw_in_docker else 'Unknown'
+ hw_product = 'Container' if hw_in_docker else 'Unknown'
+ hw_uuid = machineid.id()
+
+ if platform.system().lower() == 'darwin':
+ # Get macOS machine info
+ hw_manufacturer = 'Apple'
+ hw_product = 'Mac'
+ try:
+ # Hardware:
+ # Hardware Overview:
+ # Model Name: Mac Studio
+ # Model Identifier: Mac13,1
+ # Model Number: MJMV3LL/A
+ # ...
+ # Serial Number (system): M230YYTD77
+ # Hardware UUID: 39A12B50-1972-5910-8BEE-235AD20C8EE3
+ # ...
+ result = subprocess.run(['system_profiler', 'SPHardwareDataType'], capture_output=True, text=True, check=True)
+ for line in result.stdout.split('\n'):
+ if 'Model Name:' in line:
+ hw_product = line.split(':', 1)[-1].strip()
+ elif 'Model Identifier:' in line:
+ hw_product += ' ' + line.split(':', 1)[-1].strip()
+ elif 'Hardware UUID:' in line:
+ hw_uuid = line.split(':', 1)[-1].strip()
+ except Exception:
+ pass
+ else:
+ # get Linux machine info
+ try:
+ # Getting SMBIOS data from sysfs.
+ # SMBIOS 2.8 present.
+ # argo-1 | 2024-10-01T10:40:51Z ERR error="Incoming request ended abruptly: context canceled" connIndex=2 event=1 ingressRule=0 originService=http://archivebox:8000 │
+ # Handle 0x0100, DMI type 1, 27 bytes
+ # System Information
+ # Manufacturer: DigitalOcean
+ # Product Name: Droplet
+ # Serial Number: 411922099
+ # UUID: fb65f41c-ec24-4539-beaf-f941903bdb2c
+ # ...
+ # Family: DigitalOcean_Droplet
+ dmidecode = subprocess.run(['dmidecode', '-t', 'system'], capture_output=True, text=True, check=True)
+ for line in dmidecode.stdout.split('\n'):
+ if 'Manufacturer:' in line:
+ hw_manufacturer = line.split(':', 1)[-1].strip()
+ elif 'Product Name:' in line:
+ hw_product = line.split(':', 1)[-1].strip()
+ elif 'UUID:' in line:
+ hw_uuid = line.split(':', 1)[-1].strip()
+ except Exception:
+ pass
+
+ # Check for VM fingerprint in manufacturer/product name
+ if 'qemu' in hw_product.lower() or 'vbox' in hw_product.lower() or 'lxc' in hw_product.lower() or 'vm' in hw_product.lower():
+ hw_in_vm = True
+
+ # Check for QEMU explicitly in pmap output
+ try:
+ result = subprocess.run(['pmap', '1'], capture_output=True, text=True, check=True)
+ if 'qemu' in result.stdout.lower():
+ hw_in_vm = True
+ except Exception:
+ pass
+
+ return {
+ "hw_in_docker": hw_in_docker,
+ "hw_in_vm": hw_in_vm,
+ "hw_manufacturer": hw_manufacturer,
+ "hw_product": hw_product,
+ "hw_uuid": hw_uuid,
+ }
+
+def get_public_ip() -> str:
+ def fetch_url(url: str) -> str:
+ with urllib.request.urlopen(url, timeout=5) as response:
+ return response.read().decode('utf-8').strip()
+
+ def fetch_dns(pubip_lookup_host: str) -> str:
+ return socket.gethostbyname(pubip_lookup_host).strip()
+
+ methods = [
+ (lambda: fetch_url("https://ipinfo.io/ip"), lambda r: r),
+ (lambda: fetch_url("https://api.ipify.org?format=json"), lambda r: json.loads(r)['ip']),
+ (lambda: fetch_dns("myip.opendns.com"), lambda r: r),
+ (lambda: fetch_url("http://whatismyip.akamai.com/"), lambda r: r), # try HTTP as final fallback in case of TLS/system time errors
+ ]
+
+ for fetch, parse in methods:
+ try:
+ result = parse(fetch())
+ if result:
+ return result
+ except Exception:
+ continue
+
+ raise Exception("Could not determine public IP address")
+
+def get_local_ip(remote_ip: str='1.1.1.1', remote_port: int=80) -> str:
+ try:
+ with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
+ s.connect((remote_ip, remote_port))
+ return s.getsockname()[0]
+ except Exception:
+ pass
+ return '127.0.0.1'
+
+ip_addrs = lambda addrs: (a for a in addrs if a.family == socket.AF_INET)
+mac_addrs = lambda addrs: (a for a in addrs if a.family == psutil.AF_LINK)
+
+def get_isp_info(ip=None):
+ # Get public IP
+ try:
+ ip = ip or urllib.request.urlopen('https://api.ipify.org').read().decode('utf8')
+ except Exception:
+ pass
+
+ # Get ISP name, city, and country
+ data = {}
+ try:
+ url = f'https://ipapi.co/{ip}/json/'
+ response = urllib.request.urlopen(url)
+ data = json.loads(response.read().decode())
+ except Exception:
+ pass
+
+ isp = data.get('org', 'Unknown')
+ city = data.get('city', 'Unknown')
+ region = data.get('region', 'Unknown')
+ country = data.get('country_name', 'Unknown')
+
+ # Get system DNS resolver servers
+ dns_server = None
+ try:
+ result = subprocess.run(['dig', 'example.com', 'A'], capture_output=True, text=True, check=True).stdout
+ dns_server = result.split(';; SERVER: ', 1)[-1].split('\n')[0].split('#')[0].strip()
+ except Exception:
+ pass
+
+ # Get DNS resolver's ISP name
+ # url = f'https://ipapi.co/{dns_server}/json/'
+ # dns_isp = json.loads(urllib.request.urlopen(url).read().decode()).get('org', 'Unknown')
+
+ return {
+ 'isp': isp,
+ 'city': city,
+ 'region': region,
+ 'country': country,
+ 'dns_server': dns_server,
+ # 'net_dns_isp': dns_isp,
+ }
+
+def get_host_network() -> Dict[str, Any]:
+ default_gateway_local_ip = get_local_ip()
+ gateways = psutil.net_if_addrs()
+
+ for interface, ips in gateways.items():
+ for local_ip in ip_addrs(ips):
+ if default_gateway_local_ip == local_ip.address:
+ mac_address = next(mac_addrs(ips)).address
+ public_ip = get_public_ip()
+ return {
+ "hostname": max([socket.gethostname(), platform.node()], key=len),
+ "iface": interface,
+ "mac_address": mac_address,
+ "ip_local": local_ip.address,
+ "ip_public": public_ip,
+ # "is_behind_nat": local_ip.address != public_ip,
+ **get_isp_info(public_ip),
+ }
+
+ raise Exception("Could not determine host network info")
+
+
+def get_os_info() -> Dict[str, Any]:
+ os_release = platform.release()
+ if platform.system().lower() == 'darwin':
+ os_release = 'macOS ' + platform.mac_ver()[0]
+ else:
+ try:
+ os_release = subprocess.run(['lsb_release', '-ds'], capture_output=True, text=True, check=True).stdout.strip()
+ except Exception:
+ pass
+
+ return {
+ "os_arch": platform.machine(),
+ "os_family": platform.system().lower(),
+ "os_platform": platform.platform(),
+ "os_kernel": platform.version(),
+ "os_release": os_release,
+ }
+
+def get_host_stats() -> Dict[str, Any]:
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ tmp_usage = psutil.disk_usage(str(tmp_dir))
+ app_usage = psutil.disk_usage(str(PACKAGE_DIR))
+ data_usage = psutil.disk_usage(str(DATA_DIR))
+ mem_usage = psutil.virtual_memory()
+ swap_usage = psutil.swap_memory()
+ return {
+ "cpu_boot_time": datetime.fromtimestamp(psutil.boot_time()).isoformat(),
+ "cpu_count": psutil.cpu_count(logical=False),
+ "cpu_load": psutil.getloadavg(),
+ # "cpu_pct": psutil.cpu_percent(interval=1),
+ "mem_virt_used_pct": mem_usage.percent,
+ "mem_virt_used_gb": round(mem_usage.used / 1024 / 1024 / 1024, 3),
+ "mem_virt_free_gb": round(mem_usage.free / 1024 / 1024 / 1024, 3),
+ "mem_swap_used_pct": swap_usage.percent,
+ "mem_swap_used_gb": round(swap_usage.used / 1024 / 1024 / 1024, 3),
+ "mem_swap_free_gb": round(swap_usage.free / 1024 / 1024 / 1024, 3),
+ "disk_tmp_used_pct": tmp_usage.percent,
+ "disk_tmp_used_gb": round(tmp_usage.used / 1024 / 1024 / 1024, 3),
+ "disk_tmp_free_gb": round(tmp_usage.free / 1024 / 1024 / 1024, 3), # in GB
+ "disk_app_used_pct": app_usage.percent,
+ "disk_app_used_gb": round(app_usage.used / 1024 / 1024 / 1024, 3),
+ "disk_app_free_gb": round(app_usage.free / 1024 / 1024 / 1024, 3),
+ "disk_data_used_pct": data_usage.percent,
+ "disk_data_used_gb": round(data_usage.used / 1024 / 1024 / 1024, 3),
+ "disk_data_free_gb": round(data_usage.free / 1024 / 1024 / 1024, 3),
+ }
+
+def get_host_immutable_info(host_info: Dict[str, Any]) -> Dict[str, Any]:
+ return {
+ key: value
+ for key, value in host_info.items()
+ if key in ['guid', 'net_mac', 'os_family', 'cpu_arch']
+ }
+
+def get_host_guid() -> str:
+ return machineid.hashed_id('archivebox')
+
+# Example usage
+if __name__ == "__main__":
+ host_info = {
+ 'guid': get_host_guid(),
+ 'os': get_os_info(),
+ 'vm': get_vm_info(),
+ 'net': get_host_network(),
+ 'stats': get_host_stats(),
+ }
+ print(host_info)
+
+# {
+# 'guid': '1cd2dd279f8a854...6943f2384437991a',
+# 'os': {
+# 'os_arch': 'arm64',
+# 'os_family': 'darwin',
+# 'os_platform': 'macOS-14.6.1-arm64-arm-64bit',
+# 'os_kernel': 'Darwin Kernel Version 23.6.0: Mon Jul 29 21:14:30 PDT 2024; root:xnu-10063.141.2~1/RELEASE_ARM64_T6000',
+# 'os_release': 'macOS 14.6.1'
+# },
+# 'vm': {'hw_in_docker': False, 'hw_in_vm': False, 'hw_manufacturer': 'Apple', 'hw_product': 'Mac Studio Mac13,1', 'hw_uuid': '39A12B50-...-...-...-...'},
+# 'net': {
+# 'hostname': 'somehost.sub.example.com',
+# 'iface': 'en0',
+# 'mac_address': 'ab:cd:ef:12:34:56',
+# 'ip_local': '192.168.2.18',
+# 'ip_public': '123.123.123.123',
+# 'isp': 'AS-SONICTELECOM',
+# 'city': 'Berkeley',
+# 'region': 'California',
+# 'country': 'United States',
+# 'dns_server': '192.168.1.1'
+# },
+# 'stats': {
+# 'cpu_boot_time': '2024-09-24T21:20:16',
+# 'cpu_count': 10,
+# 'cpu_load': (2.35693359375, 4.013671875, 4.1171875),
+# 'mem_virt_used_pct': 66.0,
+# 'mem_virt_used_gb': 15.109,
+# 'mem_virt_free_gb': 0.065,
+# 'mem_swap_used_pct': 89.4,
+# 'mem_swap_used_gb': 8.045,
+# 'mem_swap_free_gb': 0.955,
+# 'disk_tmp_used_pct': 26.0,
+# 'disk_tmp_used_gb': 113.1,
+# 'disk_tmp_free_gb': 322.028,
+# 'disk_app_used_pct': 56.1,
+# 'disk_app_used_gb': 2138.796,
+# 'disk_app_free_gb': 1675.996,
+# 'disk_data_used_pct': 56.1,
+# 'disk_data_used_gb': 2138.796,
+# 'disk_data_free_gb': 1675.996
+# }
+# }
diff --git a/archivebox/machine/migrations/0001_initial.py b/archivebox/machine/migrations/0001_initial.py
new file mode 100644
index 00000000..815ed70e
--- /dev/null
+++ b/archivebox/machine/migrations/0001_initial.py
@@ -0,0 +1,144 @@
+# Generated by Django 5.1.1 on 2024-10-02 04:34
+
+import archivebox.abid_utils.models
+import charidfield.fields
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = []
+
+ operations = [
+ migrations.CreateModel(
+ name="Machine",
+ fields=[
+ (
+ "id",
+ models.UUIDField(
+ default=None,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ unique=True,
+ verbose_name="ID",
+ ),
+ ),
+ (
+ "abid",
+ charidfield.fields.CharIDField(
+ blank=True,
+ db_index=True,
+ default=None,
+ help_text="ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)",
+ max_length=30,
+ null=True,
+ prefix="mxn_",
+ unique=True,
+ ),
+ ),
+ (
+ "created_at",
+ archivebox.abid_utils.models.AutoDateTimeField(
+ db_index=True, default=None
+ ),
+ ),
+ ("modified_at", models.DateTimeField(auto_now=True)),
+ (
+ "guid",
+ models.CharField(
+ default=None, editable=False, max_length=64, unique=True
+ ),
+ ),
+ ("hostname", models.CharField(default=None, max_length=63)),
+ ("hw_in_docker", models.BooleanField(default=False)),
+ ("hw_in_vm", models.BooleanField(default=False)),
+ ("hw_manufacturer", models.CharField(default=None, max_length=63)),
+ ("hw_product", models.CharField(default=None, max_length=63)),
+ ("hw_uuid", models.CharField(default=None, max_length=255)),
+ ("os_arch", models.CharField(default=None, max_length=15)),
+ ("os_family", models.CharField(default=None, max_length=15)),
+ ("os_platform", models.CharField(default=None, max_length=63)),
+ ("os_release", models.CharField(default=None, max_length=63)),
+ ("os_kernel", models.CharField(default=None, max_length=255)),
+ ("stats", models.JSONField(default=None)),
+ ],
+ options={
+ "abstract": False,
+ },
+ ),
+ migrations.CreateModel(
+ name="NetworkInterface",
+ fields=[
+ (
+ "id",
+ models.UUIDField(
+ default=None,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ unique=True,
+ verbose_name="ID",
+ ),
+ ),
+ (
+ "abid",
+ charidfield.fields.CharIDField(
+ blank=True,
+ db_index=True,
+ default=None,
+ help_text="ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)",
+ max_length=30,
+ null=True,
+ prefix="ixf_",
+ unique=True,
+ ),
+ ),
+ (
+ "created_at",
+ archivebox.abid_utils.models.AutoDateTimeField(
+ db_index=True, default=None
+ ),
+ ),
+ ("modified_at", models.DateTimeField(auto_now=True)),
+ (
+ "mac_address",
+ models.CharField(default=None, editable=False, max_length=17),
+ ),
+ (
+ "ip_public",
+ models.GenericIPAddressField(default=None, editable=False),
+ ),
+ (
+ "ip_local",
+ models.GenericIPAddressField(default=None, editable=False),
+ ),
+ (
+ "dns_server",
+ models.GenericIPAddressField(default=None, editable=False),
+ ),
+ ("iface", models.CharField(default=None, max_length=15)),
+ ("hostname", models.CharField(default=None, max_length=63)),
+ ("isp", models.CharField(default=None, max_length=63)),
+ ("city", models.CharField(default=None, max_length=63)),
+ ("region", models.CharField(default=None, max_length=63)),
+ ("country", models.CharField(default=None, max_length=63)),
+ (
+ "machine",
+ models.ForeignKey(
+ default=None,
+ on_delete=django.db.models.deletion.CASCADE,
+ to="machine.machine",
+ ),
+ ),
+ ],
+ options={
+ "unique_together": {
+ ("machine", "ip_public", "ip_local", "mac_address", "dns_server")
+ },
+ },
+ ),
+ ]
diff --git a/archivebox/machine/migrations/__init__.py b/archivebox/machine/migrations/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/archivebox/machine/models.py b/archivebox/machine/models.py
new file mode 100644
index 00000000..1d8d390a
--- /dev/null
+++ b/archivebox/machine/models.py
@@ -0,0 +1,167 @@
+__package__ = 'archivebox.machine'
+
+import socket
+
+from django.db import models
+from archivebox.abid_utils.models import ABIDModel, ABIDField, AutoDateTimeField
+
+from .detect import get_host_guid, get_os_info, get_vm_info, get_host_network, get_host_stats
+
+CURRENT_MACHINE = None
+CURRENT_INTERFACE = None
+
+class MachineManager(models.Manager):
+ def current(self) -> 'Machine':
+ global CURRENT_MACHINE
+ if CURRENT_MACHINE:
+ return CURRENT_MACHINE
+
+ guid = get_host_guid()
+ try:
+ CURRENT_MACHINE = self.get(guid=guid)
+ return CURRENT_MACHINE
+ except self.model.DoesNotExist:
+ pass
+
+ CURRENT_MACHINE = self.model(
+ guid=guid,
+ hostname=socket.gethostname(),
+ **get_os_info(),
+ **get_vm_info(),
+ stats=get_host_stats(),
+ )
+ CURRENT_MACHINE.save()
+ return CURRENT_MACHINE
+
+class Machine(ABIDModel):
+ abid_prefix = 'mxn_'
+ abid_ts_src = 'self.created_at'
+ abid_uri_src = 'self.guid'
+ abid_subtype_src = '"01"'
+ abid_rand_src = 'self.id'
+ abid_drift_allowed = False
+
+ id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
+ abid = ABIDField(prefix=abid_prefix)
+
+ created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+ modified_at = models.DateTimeField(auto_now=True)
+
+ # IMMUTABLE PROPERTIES
+ guid = models.CharField(max_length=64, default=None, null=False, unique=True, editable=False)
+
+ # MUTABLE PROPERTIES
+ hostname = models.CharField(max_length=63, default=None, null=False)
+
+ hw_in_docker = models.BooleanField(default=False, null=False)
+ hw_in_vm = models.BooleanField(default=False, null=False)
+ hw_manufacturer = models.CharField(max_length=63, default=None, null=False) # e.g. Apple
+ hw_product = models.CharField(max_length=63, default=None, null=False) # e.g. Mac Studio Mac13,1
+ hw_uuid = models.CharField(max_length=255, default=None, null=False) # e.g. 39A12B50-...-...-...-...
+
+ os_arch = models.CharField(max_length=15, default=None, null=False) # e.g. arm64
+ os_family = models.CharField(max_length=15, default=None, null=False) # e.g. darwin
+ os_platform = models.CharField(max_length=63, default=None, null=False) # e.g. macOS-14.6.1-arm64-arm-64bit
+ os_release = models.CharField(max_length=63, default=None, null=False) # e.g. macOS 14.6.1
+ os_kernel = models.CharField(max_length=255, default=None, null=False) # e.g. Darwin Kernel Version 23.6.0: Mon Jul 29 21:14:30 PDT 2024; root:xnu-10063.141.2~1/RELEASE_ARM64_T6000
+
+ stats = models.JSONField(default=None, null=False)
+
+ objects = MachineManager()
+
+ networkinterface_set: models.Manager['NetworkInterface']
+
+
+class NetworkInterfaceManager(models.Manager):
+ def current(self) -> 'NetworkInterface':
+ global CURRENT_INTERFACE
+ if CURRENT_INTERFACE:
+ return CURRENT_INTERFACE
+
+ machine = Machine.objects.current()
+ net_info = get_host_network()
+ try:
+ CURRENT_INTERFACE = self.get(
+ machine=machine,
+ ip_public=net_info['ip_public'],
+ ip_local=net_info['ip_local'],
+ mac_address=net_info['mac_address'],
+ dns_server=net_info['dns_server'],
+ )
+ return CURRENT_INTERFACE
+ except self.model.DoesNotExist:
+ pass
+
+ CURRENT_INTERFACE = self.model(
+ machine=machine,
+ **get_host_network(),
+ )
+ CURRENT_INTERFACE.save()
+ return CURRENT_INTERFACE
+
+
+
+class NetworkInterface(ABIDModel):
+ abid_prefix = 'ixf_'
+ abid_ts_src = 'self.machine.created_at'
+ abid_uri_src = 'self.machine.guid'
+ abid_subtype_src = 'self.iface'
+ abid_rand_src = 'self.id'
+ abid_drift_allowed = False
+
+ id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
+ abid = ABIDField(prefix=abid_prefix)
+
+ created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+ modified_at = models.DateTimeField(auto_now=True)
+
+ machine = models.ForeignKey(Machine, on_delete=models.CASCADE, default=None, null=False)
+
+ # IMMUTABLE PROPERTIES
+ mac_address = models.CharField(max_length=17, default=None, null=False, editable=False) # e.g. ab:cd:ef:12:34:56
+ ip_public = models.GenericIPAddressField(default=None, null=False, editable=False) # e.g. 123.123.123.123 or 2001:0db8:85a3:0000:0000:8a2e:0370:7334
+ ip_local = models.GenericIPAddressField(default=None, null=False, editable=False) # e.g. 192.168.2.18 or 2001:0db8:85a3:0000:0000:8a2e:0370:7334
+ dns_server = models.GenericIPAddressField(default=None, null=False, editable=False) # e.g. 8.8.8.8 or 2001:0db8:85a3:0000:0000:8a2e:0370:7334
+
+ # MUTABLE PROPERTIES
+ iface = models.CharField(max_length=15, default=None, null=False) # e.g. en0
+ hostname = models.CharField(max_length=63, default=None, null=False) # e.g. somehost.sub.example.com
+ isp = models.CharField(max_length=63, default=None, null=False) # e.g. AS-SONICTELECOM
+ city = models.CharField(max_length=63, default=None, null=False) # e.g. Berkeley
+ region = models.CharField(max_length=63, default=None, null=False) # e.g. California
+ country = models.CharField(max_length=63, default=None, null=False) # e.g. United States
+
+ objects = NetworkInterfaceManager()
+
+ class Meta:
+ unique_together = (
+ ('machine', 'ip_public', 'ip_local', 'mac_address', 'dns_server'),
+ )
+
+
+# class InstalledBinary(ABIDModel):
+# abid_prefix = 'bin_'
+# abid_ts_src = 'self.machine.created_at'
+# abid_uri_src = 'self.machine.guid'
+# abid_subtype_src = 'self.binprovider'
+# abid_rand_src = 'self.id'
+# abid_drift_allowed = False
+
+# id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
+# abid = ABIDField(prefix=abid_prefix)
+
+# created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+# modified_at = models.DateTimeField(auto_now=True)
+
+# machine = models.ForeignKey(Machine, on_delete=models.CASCADE, default=None, null=False)
+# binprovider = models.CharField(max_length=255, default=None, null=False)
+
+# name = models.CharField(max_length=255, default=None, null=False)
+# version = models.CharField(max_length=255, default=None, null=False)
+# abspath = models.CharField(max_length=255, default=None, null=False)
+# sha256 = models.CharField(max_length=255, default=None, null=False)
+
+# class Meta:
+# unique_together = (
+# ('machine', 'binprovider', 'version', 'abspath', 'sha256'),
+# )
diff --git a/pdm.lock b/pdm.lock
index 50dea8a6..9acb852e 100644
--- a/pdm.lock
+++ b/pdm.lock
@@ -5,7 +5,7 @@
groups = ["default", "all", "ldap", "sonic"]
strategy = ["inherit_metadata"]
lock_version = "4.5.0"
-content_hash = "sha256:cdf785c77dcdb8927b7743c36374dc5f2377db78622d27eb8356648d61275a0a"
+content_hash = "sha256:3c924966bd7b6d20a3e653f83b72f7c4160088f136e0d4621650c96b23f75803"
[[metadata.targets]]
requires_python = "==3.11.*"
@@ -13,7 +13,7 @@ platform = "manylinux_2_17_x86_64"
[[metadata.targets]]
requires_python = "==3.11.*"
-platform = "macos_12_0_arm64"
+platform = "macos_14_0_arm64"
[[package]]
name = "annotated-types"
@@ -713,7 +713,7 @@ files = [
[[package]]
name = "httpcore"
-version = "1.0.5"
+version = "1.0.6"
requires_python = ">=3.8"
summary = "A minimal low-level HTTP client."
groups = ["default"]
@@ -723,8 +723,8 @@ dependencies = [
"h11<0.15,>=0.13",
]
files = [
- {file = "httpcore-1.0.5-py3-none-any.whl", hash = "sha256:421f18bac248b25d310f3cacd198d55b8e6125c107797b609ff9b7a6ba7991b5"},
- {file = "httpcore-1.0.5.tar.gz", hash = "sha256:34a38e2f9291467ee3b44e89dd52615370e152954ba21721378a87b2960f7a61"},
+ {file = "httpcore-1.0.6-py3-none-any.whl", hash = "sha256:27b59625743b85577a8c0e10e55b50b5368a4f2cfe8cc7bcfa9cf00829c2682f"},
+ {file = "httpcore-1.0.6.tar.gz", hash = "sha256:73f6dbd6eb8c21bbf7ef8efad555481853f5f6acdeaff1edb0694289269ee17f"},
]
[[package]]
@@ -842,12 +842,12 @@ files = [
[[package]]
name = "mailchecker"
-version = "6.0.9"
+version = "6.0.10"
summary = "Cross-language email validation. Backed by a database of thousands throwable email providers."
groups = ["default"]
marker = "python_version == \"3.11\""
files = [
- {file = "mailchecker-6.0.9.tar.gz", hash = "sha256:f17e907ffe6f6faedc243f57eb0c9c951f61dec9af8e96922c1dcd093389b88d"},
+ {file = "mailchecker-6.0.10.tar.gz", hash = "sha256:d933fecb90a66459c8aa543a272890f97c02f6cbf30a3f5016ce2a1699848bee"},
]
[[package]]
@@ -993,19 +993,6 @@ dependencies = [
"requests",
]
-[[package]]
-name = "pocket"
-version = "0.3.7"
-git = "https://github.com/tapanpandita/pocket.git"
-ref = "v0.3.7"
-revision = "5a144438cc89bfc0ec94db960718ccf1f76468c1"
-summary = "api wrapper for getpocket.com"
-groups = ["default"]
-marker = "python_version == \"3.11\""
-dependencies = [
- "requests",
-]
-
[[package]]
name = "prompt-toolkit"
version = "3.0.48"
@@ -1056,6 +1043,20 @@ files = [
{file = "pure_eval-0.2.3.tar.gz", hash = "sha256:5f4e983f40564c576c7c8635ae88db5956bb2229d7e9237d03b3c0b0190eaf42"},
]
+[[package]]
+name = "py-machineid"
+version = "0.6.0"
+summary = "Get the unique machine ID of any host (without admin privileges)"
+groups = ["default"]
+marker = "python_version == \"3.11\""
+dependencies = [
+ "winregistry; sys_platform == \"win32\"",
+]
+files = [
+ {file = "py-machineid-0.6.0.tar.gz", hash = "sha256:00c38d8521d429a4539bdd92967234db28a1a2b4b263062b351ca002332e633f"},
+ {file = "py_machineid-0.6.0-py3-none-any.whl", hash = "sha256:63214f8a98737311716b29d279716dc121a6495f16486caf5c032433f81cdfd6"},
+]
+
[[package]]
name = "pyasn1"
version = "0.6.1"
@@ -1144,7 +1145,7 @@ files = [
[[package]]
name = "pydantic-pkgr"
-version = "0.3.7"
+version = "0.3.8"
requires_python = ">=3.10"
summary = "System package manager APIs in strongly typed Python"
groups = ["default"]
@@ -1155,8 +1156,8 @@ dependencies = [
"typing-extensions>=4.11.0",
]
files = [
- {file = "pydantic_pkgr-0.3.7-py3-none-any.whl", hash = "sha256:fdb63b2cee79d7c9d53673b9d61afa846921fd4950a8c16a8c4d2555cd0f6478"},
- {file = "pydantic_pkgr-0.3.7.tar.gz", hash = "sha256:6e575cdc3584d375eb8d5024e5e8bade1c225c2aee3af1a076951dbc1a2c1f2d"},
+ {file = "pydantic_pkgr-0.3.8-py3-none-any.whl", hash = "sha256:fefa34449feb8fc09d73d6beb8a61afe5959b1a848f0a5bba9db1d092d7099be"},
+ {file = "pydantic_pkgr-0.3.8.tar.gz", hash = "sha256:5ca12f4ee1c82ce0a2231c36b898534899a40a9e77cc4c97175fac9d1dc6e351"},
]
[[package]]
@@ -1427,19 +1428,19 @@ files = [
[[package]]
name = "rich"
-version = "13.8.1"
-requires_python = ">=3.7.0"
+version = "13.9.1"
+requires_python = ">=3.8.0"
summary = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal"
groups = ["default"]
marker = "python_version == \"3.11\""
dependencies = [
"markdown-it-py>=2.2.0",
"pygments<3.0.0,>=2.13.0",
- "typing-extensions<5.0,>=4.0.0; python_version < \"3.9\"",
+ "typing-extensions<5.0,>=4.0.0; python_version < \"3.11\"",
]
files = [
- {file = "rich-13.8.1-py3-none-any.whl", hash = "sha256:1760a3c0848469b97b558fc61c85233e3dafb69c7a071b4d60c38099d3cd4c06"},
- {file = "rich-13.8.1.tar.gz", hash = "sha256:8260cda28e3db6bf04d2d1ef4dbc03ba80a824c88b0e7668a0f23126a424844a"},
+ {file = "rich-13.9.1-py3-none-any.whl", hash = "sha256:b340e739f30aa58921dc477b8adaa9ecdb7cecc217be01d93730ee1bc8aa83be"},
+ {file = "rich-13.9.1.tar.gz", hash = "sha256:097cffdf85db1babe30cc7deba5ab3a29e1b9885047dab24c57e9a7f8a9c1466"},
]
[[package]]
@@ -1815,7 +1816,7 @@ files = [
[[package]]
name = "yt-dlp"
-version = "2024.8.6"
+version = "2024.9.27"
requires_python = ">=3.8"
summary = "A feature-rich command-line audio/video downloader"
groups = ["default"]
@@ -1828,11 +1829,11 @@ dependencies = [
"pycryptodomex",
"requests<3,>=2.32.2",
"urllib3<3,>=1.26.17",
- "websockets>=12.0",
+ "websockets>=13.0",
]
files = [
- {file = "yt_dlp-2024.8.6-py3-none-any.whl", hash = "sha256:ab507ff600bd9269ad4d654e309646976778f0e243eaa2f6c3c3214278bb2922"},
- {file = "yt_dlp-2024.8.6.tar.gz", hash = "sha256:e8551f26bc8bf67b99c12373cc87ed2073436c3437e53290878d0f4b4bb1f663"},
+ {file = "yt_dlp-2024.9.27-py3-none-any.whl", hash = "sha256:2717468dd697fcfcf9a89f493ba30a3830cdfb276c09750e5b561b08b9ef5f69"},
+ {file = "yt_dlp-2024.9.27.tar.gz", hash = "sha256:86605542e17e2e23ad23145b637ec308133762a15a5dedac4ae50b7973237026"},
]
[[package]]
diff --git a/pyproject.toml b/pyproject.toml
index 3313933b..434f8e8f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -86,6 +86,7 @@ dependencies = [
"base32-crockford==0.3.0",
############# Extractor Dependencies #############
"yt-dlp>=2024.8.6", # for: media
+ "py-machineid>=0.6.0",
]
# pdm lock --group=':all'
diff --git a/requirements.txt b/requirements.txt
index 4ca23b98..25feaaf1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -46,7 +46,7 @@ executing==2.1.0; python_version == "3.11"
feedparser==6.0.11; python_version == "3.11"
ftfy==6.2.3; python_version == "3.11"
h11==0.14.0; python_version == "3.11"
-httpcore==1.0.5; python_version == "3.11"
+httpcore==1.0.6; python_version == "3.11"
httpx==0.27.2; python_version == "3.11"
huey==2.5.2; python_version == "3.11"
hyperlink==21.0.0; python_version == "3.11"
@@ -54,7 +54,7 @@ idna==3.10; python_version == "3.11"
incremental==24.7.2; python_version == "3.11"
ipython==8.27.0; python_version == "3.11"
jedi==0.19.1; python_version == "3.11"
-mailchecker==6.0.9; python_version == "3.11"
+mailchecker==6.0.10; python_version == "3.11"
markdown-it-py==3.0.0; python_version == "3.11"
matplotlib-inline==0.1.7; python_version == "3.11"
mdurl==0.1.2; python_version == "3.11"
@@ -76,7 +76,7 @@ pycparser==2.22; platform_python_implementation != "PyPy" and python_version ==
pycryptodomex==3.20.0; python_version == "3.11"
pydantic==2.9.2; python_version == "3.11"
pydantic-core==2.23.4; python_version == "3.11"
-pydantic-pkgr==0.3.7; python_version == "3.11"
+pydantic-pkgr==0.3.8; python_version == "3.11"
pydantic-settings==2.5.2; python_version == "3.11"
pygments==2.18.0; python_version == "3.11"
pyopenssl==24.2.1; python_version == "3.11"
@@ -94,7 +94,7 @@ pytz==2024.2; python_version == "3.11"
pyyaml==6.0.2; python_version == "3.11"
regex==2024.9.11; python_version == "3.11"
requests==2.32.3; python_version == "3.11"
-rich==13.8.1; python_version == "3.11"
+rich==13.9.1; python_version == "3.11"
service-identity==24.1.0; python_version == "3.11"
setuptools==75.1.0; python_version == "3.11"
sgmllib3k==1.0.0; python_version == "3.11"
@@ -122,5 +122,5 @@ wcwidth==0.2.13; python_version == "3.11"
websockets==13.1; python_version == "3.11"
xlrd==2.0.1; python_version == "3.11"
xmltodict==0.13.0; python_version == "3.11"
-yt-dlp==2024.8.6; python_version == "3.11"
+yt-dlp==2024.9.27; python_version == "3.11"
zope-interface==7.0.3; python_version == "3.11"