Merge branch 'main' into workflow-deploy-no-makefile

cweider · Oct 21, 2024 · fb269c1 · fb269c1
2 parents b3f3aef + d1fc891
commit fb269c1
Show file tree

Hide file tree

Showing 573 changed files with 57,079 additions and 15,766 deletions.
diff --git a/.editorconfig b/.editorconfig
@@ -1,4 +1,4 @@
-# http://editorconfig.org
+# https://editorconfig.org
 
 root = true
 

diff --git a/.env.example b/.env.example
@@ -2,14 +2,6 @@
 
 # money.py
 EIN_SECRET=""
-PAYPAL_DEV_CLIENT_ID=""
-PAYPAL_DEV_SECRET_KEY=""
-PAYPAL_CLIENT_ID=""
-PAYPAL_SECRET_KEY=""
-STRIPE_DEV_SECRET_KEY=""
-STRIPE_DEV_PUBLIC_KEY=""
-STRIPE_SECRET_KEY=""
-STRIPE_PUBLIC_KEY=""
 
 # security.py
 #ALLOWED_HOSTS="*"  # <-- Uncomment this in dev
@@ -28,9 +20,6 @@ AWS_LAMBDA_PROXY_URL=""
 HCAPTCHA_SITEKEY=""
 HCAPTCHA_SECRET=""
 
-# moosend.py
-MOOSEND_API_KEY=""
-
 # sentry.py
 SENTRY_DSN=""
 SENTRY_REPORT_URI=""
@@ -53,3 +42,6 @@ IA_ACCESS_KEY=""
 IA_SECRET_KEY=""
 
 FTM_KEY=""
+
+# CL API key for cloning data
+CL_API_KEY=""
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -25,4 +25,4 @@ updates:
   groups:
     core:
       patterns:
-        - 'actions/*' # The core actions offered by Github
+        - 'actions/*' # The core actions offered by GitHub
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -24,15 +24,14 @@ jobs:
     # `github/codeql-action/init`)
     - uses: actions/setup-python@v5
       with:
-        python-version: '3.12'
+        python-version: '3.13'
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: python
         config-file: ./.github/codeql-config.yml
-        setup-python-dependencies: true
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3
diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
@@ -18,7 +18,7 @@ jobs:
       - name: Set up docker Buildx
         uses: docker/setup-buildx-action@v2
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -48,14 +48,15 @@ jobs:
   deploy:
     needs: build
     runs-on: ubuntu-latest
+    concurrency: production
     steps:
       - uses: actions/checkout@v4
       - name: Set shortcode
         id: vars
         run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
 
       - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v3
+        uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -64,6 +65,11 @@ jobs:
       - name: Create Kubeconfig with AWS CLI
         run: aws eks update-kubeconfig --region ${{ env.AWS_REGION }} --name ${{ env.EKS_CLUSTER_NAME }}
 
+      - name: Update Environment Variables
+        env:
+          CL_ENV: cl-env
+        run: kubectl annotate es $CL_ENV force-sync=$(date +%s) --overwrite -n ${{ env.EKS_NAMESPACE }} && kubectl wait es -n ${{ env.EKS_NAMESPACE }} --for=jsonpath="{.status.conditions[?(@.reason=='SecretSynced')].status}"=True --timeout=30s $CL_ENV
+
       - name: Launch Temporary Pod
         id: tempPod
         run: |
@@ -123,6 +129,11 @@ jobs:
       - name: Watch cl-celery-prefork-bulk rollout status
         run: kubectl rollout status -n ${{ env.EKS_NAMESPACE }} deployment/cl-celery-prefork-bulk
 
+      - name: Rollout cl-celery-prefork-es-sweep
+        run: kubectl set image -n ${{ env.EKS_NAMESPACE }} deployment/cl-celery-prefork-es-sweep cl-celery-prefork-es-sweep=freelawproject/courtlistener:${{ steps.vars.outputs.sha_short }}-prod
+      - name: Watch cl-celery-prefork-es-sweep rollout status
+        run: kubectl rollout status -n ${{ env.EKS_NAMESPACE }} deployment/cl-celery-prefork-es-sweep
+
       - name: Rollout cl-scrape-rss
         run: kubectl set image -n ${{ env.EKS_NAMESPACE }} deployment/cl-scrape-rss scrape-rss=freelawproject/courtlistener@${{ needs.build.outputs.digest }}
       - name: Watch cl-scrape-rss rollout status
@@ -133,6 +144,12 @@ jobs:
       - name: Watch cl-retry-webhooks rollout status
         run: kubectl rollout status -n ${{ env.EKS_NAMESPACE }} deployment/cl-retry-webhooks
 
+      - name: Rollout cl-es-sweep-indexer
+        run: kubectl set image -n ${{ env.EKS_NAMESPACE }} deployment/cl-es-sweep-indexer sweep-indexer=freelawproject/courtlistener:${{ steps.vars.outputs.sha_short }}-prod
+      - name: Watch cl-es-sweep-indexer rollout status
+        run: kubectl rollout status -n ${{ env.EKS_NAMESPACE }} deployment/cl-es-sweep-indexer
+
+
       # Watch "cronjobs" in k9s
       - name: Update cronjobs
         run: |

diff --git a/.github/workflows/flp-dependencies-pr.yml b/.github/workflows/flp-dependencies-pr.yml
@@ -8,10 +8,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: '3.x'
 
@@ -24,7 +24,7 @@ jobs:
           poetry update courts-db eyecite juriscraper reporters-db
 
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v3
+        uses: peter-evans/create-pull-request@v7
         with:
           commit-message: Update freelawproject dependencies
           title: Update freelawproject dependencies

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -17,12 +17,12 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           # Once codebase is updated, this can easily be changed to any specific version.
-          python-version: "3.12"
+          python-version: "3.13"
 
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
-          version: 1.7.1
+          version: 1.8.4
           virtualenvs-create: true
           virtualenvs-in-project: true
 
@@ -32,7 +32,7 @@ jobs:
 
       - name: Load cached venv
         id: cached-poetry-dependencies
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: .venv
           key: venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('**/poetry.lock')}}-v20210414
@@ -70,7 +70,7 @@ jobs:
           -p cl.simple_pages
 
       # Add files here as they're ready
-      - name: mypy Static Type Cheker (files)
+      - name: mypy Static Type Checker (files)
         run: |
           mypy \
           --follow-imports=skip \
@@ -84,16 +84,16 @@ jobs:
           cl/api/utils.py \
           cl/api/views.py \
           cl/api/webhooks.py \
-          cl/donate/management/commands/charge_monthly_donors.py \
-          cl/donate/utils.py \
           cl/tests/utils.py \
           cl/users/email_handlers.py \
           cl/users/forms.py \
           cl/users/management/commands/cl_account_management.py \
           cl/users/management/commands/cl_delete_old_emails.py \
           cl/users/management/commands/cl_retry_failed_email.py \
           cl/users/tasks.py \
-          cl/recap/management/commands/remove_appellate_entries_with_long_numbers.py
+          cl/recap/management/commands/remove_appellate_entries_with_long_numbers.py \
+          cl/search/management/commands/cl_index_parent_and_child_docs.py \
+          cl/search/management/commands/sweep_indexer.py
 
 
 

diff --git a/.github/workflows/sentry-release.yml b/.github/workflows/sentry-release.yml
@@ -11,7 +11,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Create a Sentry.io release
-        uses: getsentry/action-release@v1.4.1
+        uses: getsentry/action-release@v1.7.0
         env:
           SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
           SENTRY_ORG: freelawproject

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -53,11 +53,11 @@ jobs:
       # Build and cache docker images so tests are always run on the latest
       # dependencies
       - name: Set up docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
         with:
           driver-opts: network=host
       - name: Prebuild docker images
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v6
         with:
           context: ./courtlistener
           file: ./courtlistener/docker/django/Dockerfile
@@ -97,21 +97,22 @@ jobs:
           docker exec -e SELENIUM_DEBUG=1 -e SELENIUM_TIMEOUT=30 cl-django
           python /opt/courtlistener/manage.py test
           cl --verbosity=2 ${{ matrix.tag_flags }} --parallel
-      - name: cp selenium results from docker to host
+      - name: Export selenium results from docker to host
         if: failure()
         run: |
           # This is annoying b/c docker cp doesn't support globs. See:
           # https://stackoverflow.com/q/35806102/
           # https://github.com/moby/moby/issues/7710
           mkdir selenium-screenshots
-          docker exec cl-django bash -c "mkdir /extract && mv /tmp/*-selenium.png /extract"
+          docker exec cl-django bash -c "mkdir /extract && mv /tmp/*-selenium.png /extract ||:"
           docker cp 'cl-django:/extract' selenium-screenshots/
       - name: Save selenium screenshot as Github artifacts
         uses: actions/upload-artifact@master
         if: failure()
         with:
           name: selenium-screenshots
           path: selenium-screenshots/extract
+          if-no-files-found: ignore
 
 # Cancel the current workflow (tests) for pull requests (head_ref) only. See:
 # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#example-using-a-fallback-value

diff --git a/.gitignore b/.gitignore
@@ -33,6 +33,9 @@ cl/assets/static-global/js/react
 # IntelliJ IDEA files
 .idea
 
+# Hyperscan db
+.hyperscan/
+
 # Dev personal stuff - never check in these things with keys or codes!
 cl/settings/*private.py
 cl/settings/google_auth.json
@@ -41,5 +44,6 @@ cl/settings/google_auth.json
 /.env.dev
 /.env
 
+
 .vscode/*
 
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -2,12 +2,12 @@
 # See https://pre-commit.com/hooks.html for more hooks
 exclude: migrations
 default_language_version:
-  python: "python3.12"
+  python: "python3.13"
 ci:
   autoupdate_schedule: quarterly
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
      - id: check-added-large-files
      - id: check-ast
@@ -33,7 +33,7 @@ repos:
        args: [--line-length=79, --transform-concats]
 
   - repo: https://github.com/psf/black
-    rev: 24.1.0
+    rev: 24.4.2
     hooks:
      - id: black
 

diff --git a/LICENSE.txt b/LICENSE.txt
@@ -16,7 +16,7 @@ GNU Affero General Public License below, for more details.
                     GNU AFFERO GENERAL PUBLIC LICENSE
                        Version 3, 19 November 2007
 
- Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 
@@ -658,7 +658,7 @@ the "copyright" line and a pointer to where the full notice is found.
     GNU Affero General Public License for more details.
 
     You should have received a copy of the GNU Affero General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 Also add information on how to contact you by electronic and paper mail.
 
@@ -673,4 +673,4 @@ specific requirements.
   You should also get your employer (if you work as a programmer) or school,
 if any, to sign a "copyright disclaimer" for the program, if necessary.
 For more information on this, and how to apply and follow the GNU AGPL, see
-<http://www.gnu.org/licenses/>.
+<https://www.gnu.org/licenses/>.
diff --git a/README.md b/README.md
@@ -40,22 +40,55 @@ To contact Free Law Project, see here:
 https://free.law/contact/
 
 
-                       ````
-                .:+oo++//++osso+/. -+++////+++.
-             -+ys/-`         ./yy+  `./mmmm/``
-           -sys:               `oo     ymmy
-          +yyo`                 `+`    ymmy
-         +yyy`                         ymms
-        -yyy+                          ymms
-        +yyy:                          ymms
-        +sss:                          ymms
-        /sss+                          ydds
-        `ssss.                         sdds
-         -syyo`                  ``    sdds
-          .oyys-                `s/    ydds            `+`
-            :shhs:`           `/ys`    yddh`          .hs
-              .:oyys+:-....-/oyys.  `./ddddy/:--.---:odd.
-                  `.-::///::-.`    -///////////////////-
+```
+                                   g@@D
+                                  "l@@B!
+                                   "@@"
+                                    @@
+                                    @@
+                            _P '@.  @@
+                            71__@   @@
+                              @@    @@    __
+                              @@    @@  ;F  @
+                              @@    @@  'h__@
+                              @@    @@    @g
+                              @@    @@    @g
+                              @@    @@    @g                     _~~_
+                              @@    @@    @g   @@@@@@@@@@@@@@@@@@F  |!
+                              @@    @@    @g   @@         @T     TmmP
+   _gg_                       @@    @@    @g   @@         @'
+   @   @gggggggggggggggggg    @@    @@    @g   @@         @\
+   '@WP      .@         @@    @@    @@    @g   @@        J "_
+             !@         @@    @@    @@    @g   @@       ,'  T
+             ;@         @@    @@    @@    @g   @@       8    %
+             W @        @@    @@    @@    @g   @@   ___d______@_-_
+            @   q       @@    @@    @@    @g   @@   ______________
+           ;"    g      @@    @@    @@    @g   @@   0@@@@@@@@@@@@"
+       ____E_____]L___  @@    @@    @@    @g   @@
+       ,_____________   @@    @@    @@    @g   @@
+       '@@@@@@@@@@@@D   @@    @@    @@    @g   @@
+                        @@    @@    @@    @g   @@
+                        @@    @@    @@    @g   @@
+                  _~ggg~_.       __g@g~_.      gg        ggg   gggggggg_,   ;gggggggggggg
+                g@@P"""<@@g    _@@P"""<B@g     @@        @@@   @@@"""""Q@g  """""9@g"""""
+              .@@F       "    @@F       "@@,   @@        @@@   @@@      @@g      [@g
+              g@@            |@@         (@@   @@        @@@   @@@     ,@@/      [@g
+              [@@            [@@         j@@   @@        @@g   @@@@@@@@@B        [@g
+               @@L       ,    @@L       ,@@'   @@1       @@'   @@@   '@@L        [@g
+                T@@_____g@@    T@@_____g@@      @@g____+@@?    @@@     @@a       [@g
+                  '4B@BP"        '=B@BP"          <8B@B+"      BBB      0BB      "BN
+              g        ;;   _~mma_  mmmmqmmmmm  mmmmmmms  _        ;   gmmmmmmm  gmmmmm__
+              g        [|  F            |]      |         g\_      [   @         @       q
+              g        [|  1.           |]      |         g  q     [   @         @       [
+              g        [|    "+m__      |]      P""""""   g   "_   [   @""""""   @_______'
+              g        [|         \,    |]      |         g     \, [   @         @    `a
+              g        [| ,       /'    |]      |         g       q[   @         @      0
+              """""""" ''   ""==""      '"      """"""""  "        "   """"""""  "       "
+                        @@    @@    @@    @g   @@
+                        @@    @@    @@    @g   @@
+                        @@    @@    @@    @g   @@
+```
+
 
 
 [issues]: https://github.com/freelawproject/courtlistener/issues