package and deliver parser as a library (#114)

* package and deliver parser as a library

* bye arrow

* apply black formatting

Author: ewdurbin

.github/workflows/release.yml

@@ -0,0 +1,35 @@
+name: Publish to PyPI 
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+   pypi-publish:
+     name: upload release to PyPI
+     runs-on: ubuntu-latest
+     environment: release
+     permissions:
+       id-token: write
+     steps:
+       - uses: actions/checkout@v3
+       - uses: actions/setup-python@v4
+         with:
+           python-version: '3.11' 
+       - name: Install pypa/build
+         run: >-
+           python3 -m
+           pip install
+           build
+           --user
+       - name: Build a binary wheel and a source tarball
+         run: >-
+           python3 -m
+           build
+           --sdist
+           --wheel
+           --outdir dist/
+           .
+       - name: Publish package distributions to PyPI
+         uses: pypa/gh-action-pypi-publish@release/v1

linehaul/events/parser.py

@@ -14,9 +14,9 @@
 import logging
 import posixpath
 
+from datetime import datetime, timezone
 from typing import Optional
 
-import arrow
 import attr
 import attr.validators
 import cattr
@@ -33,7 +33,10 @@
 
 _cattr = cattr.Converter()
 _cattr.register_structure_hook(
-    arrow.Arrow, lambda d, t: arrow.get(d[5:-4], "DD MMM YYYY HH:mm:ss")
+    datetime,
+    lambda d, t: datetime.strptime(d[5:-4], "%d %b %Y %H:%M:%S").replace(
+        tzinfo=timezone.utc
+    ),
 )
 
 
@@ -145,7 +148,6 @@ class PackageType(enum.Enum):
 
 @attr.s(slots=True, frozen=True)
 class File:
-
     filename = attr.ib(validator=attr.validators.instance_of(str))
     project = attr.ib(validator=attr.validators.instance_of(str))
     version = attr.ib(validator=attr.validators.instance_of(str))
@@ -154,8 +156,7 @@ class File:
 
 @attr.s(slots=True, frozen=True)
 class Download:
-
-    timestamp = attr.ib(type=arrow.Arrow)
+    timestamp = attr.ib(type=datetime)
     url = attr.ib(validator=attr.validators.instance_of(str))
     project = attr.ib(validator=attr.validators.instance_of(str))
     file = attr.ib(type=File)
@@ -176,8 +177,7 @@ class Download:
 
 @attr.s(slots=True, frozen=True)
 class Simple:
-
-    timestamp = attr.ib(type=arrow.Arrow)
+    timestamp = attr.ib(type=datetime)
     url = attr.ib(validator=attr.validators.instance_of(str))
     project = attr.ib(validator=attr.validators.instance_of(str))
     tls_protocol = attr.ib(

linehaul/ua/datastructures.py

@@ -17,28 +17,24 @@
 
 @attr.s(slots=True, frozen=True)
 class Installer:
-
     name = attr.ib(type=Optional[str], default=None)
     version = attr.ib(type=Optional[str], default=None)
 
 
 @attr.s(slots=True, frozen=True)
 class Implementation:
-
     name = attr.ib(type=Optional[str], default=None)
     version = attr.ib(type=Optional[str], default=None)
 
 
 @attr.s(slots=True, frozen=True)
 class LibC:
-
     lib = attr.ib(type=Optional[str], default=None)
     version = attr.ib(type=Optional[str], default=None)
 
 
 @attr.s(slots=True, frozen=True)
 class Distro:
-
     name = attr.ib(type=Optional[str], default=None)
     version = attr.ib(type=Optional[str], default=None)
     id = attr.ib(type=Optional[str], default=None)
@@ -47,14 +43,12 @@ class Distro:
 
 @attr.s(slots=True, frozen=True)
 class System:
-
     name = attr.ib(type=Optional[str], default=None)
     release = attr.ib(type=Optional[str], default=None)
 
 
 @attr.s(slots=True, frozen=True)
 class UserAgent:
-
     installer = attr.ib(type=Optional[Installer], default=None)
     python = attr.ib(type=Optional[str], default=None)
     implementation = attr.ib(type=Optional[Implementation], default=None)

main.py

@@ -1,4 +1,3 @@
-import arrow
 import cattr
 
 import base64
@@ -18,7 +17,7 @@
 
 _cattr = cattr.Converter()
 _cattr.register_unstructure_hook(
-    arrow.Arrow, lambda o: o.format("YYYY-MM-DD HH:mm:ss ZZ")
+    datetime.datetime, lambda o: o.strftime("%Y-%m-%d %H:%M:%S +00:00")
 )
 
 DEFAULT_PROJECT = os.environ.get("GCP_PROJECT", "the-psf")
@@ -64,7 +63,7 @@ def process_fastly_log(data, context):
         simple_results_file = stack.enter_context(NamedTemporaryFile())
         download_results_file = stack.enter_context(NamedTemporaryFile())
 
-        min_timestamp = arrow.utcnow()
+        min_timestamp = datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc)
         for line in input_file:
             try:
                 res = parse(line.decode())

pyproject.toml

@@ -0,0 +1,37 @@
+[project]
+name = "linehaul"
+version = "1.0.0"
+description = "User-Agent parsing for PyPI analytics"
+
+readme = "README.md"
+requires-python = ">=3.11, <4"
+license = {file = "LICENSE.txt"}
+
+keywords = ["pypi", "user-agent", "parsing"]
+authors = [
+  {name = "PyPI Admins", email = "[email protected]" }
+]
+
+classifiers = [
+  "Development Status :: 5 - Production/Stable",
+  "Intended Audience :: Developers",
+  "Topic :: Software Development :: Build Tools",
+  "License :: OSI Approved :: Apache Software License",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3 :: Only",
+]
+
+dependencies = [
+    "cattrs",
+    "packaging",
+    "pyparsing",
+]
+
+[project.urls]
+"Homepage" = "https://github.com/pypi/linehaul-cloud-function"
+"Source" = "https://github.com/pypi/linehaul-cloud-function"
+
+[build-system]
+requires = ["setuptools>=43.0.0", "wheel"]
+build-backend = "setuptools.build_meta"

requirements.in

@@ -1,4 +1,3 @@
-arrow
 cattrs
 packaging
 pyparsing

requirements.txt

@@ -4,10 +4,6 @@
 #
 #    pip-compile --allow-unsafe --generate-hashes --output-file=requirements.txt requirements.in
 #
-arrow==1.2.3 \
-    --hash=sha256:3934b30ca1b9f292376d9db15b19446088d12ec58629bc3f0da28fd55fb633a1 \
-    --hash=sha256:5a49ab92e3b7b71d96cd6bfcc4df14efefc9dfa96ea19045815914a6ab6b1fe2
-    # via -r requirements.in
 attrs==22.2.0 \
     --hash=sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836 \
     --hash=sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99
@@ -348,7 +344,6 @@ python-dateutil==2.8.2 \
     --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \
     --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9
     # via
-    #   arrow
     #   google-cloud-bigquery
 requests==2.28.2 \
     --hash=sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa \

setup.cfg

@@ -0,0 +1,2 @@
+[options]
+packages = find: