Skip to content

Add left-pane file tree view and related templates #1704

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 12 commits into
base: main
Choose a base branch
from
24 changes: 24 additions & 0 deletions scanpipe/migrations/0075_codebaseresource_parent_path_and_more.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# Generated by Django 5.1.9 on 2025-06-19 21:19

from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
('scanpipe', '0074_discovered_license_models'),
]

operations = [
migrations.AddField(
model_name='codebaseresource',
name='parent_path',
field=models.CharField(blank=True, help_text="The path of the resource's parent directory. Set to None for top-level (root) resources. Used to efficiently retrieve a directory's contents.", max_length=2000, null=True),
),
migrations.AddIndex(
model_name='codebaseresource',
index=models.Index(fields=['project', 'parent_path'], name='scanpipe_co_project_008448_idx'),
),
]


34 changes: 32 additions & 2 deletions scanpipe/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
from django.db import transaction
from django.db.models import Case
from django.db.models import Count
from django.db.models import Exists
from django.db.models import IntegerField
from django.db.models import OuterRef
from django.db.models import Prefetch
Expand Down Expand Up @@ -229,7 +230,7 @@ def delete(self, *args, **kwargs):
Note that projects with queued or running pipeline runs cannot be deleted.
See the `_raise_if_run_in_progress` method.
The following if statements should not be triggered unless the `.delete()`
method is directly call from an instance of this class.
method is directly call from a instance of this class.
"""
with suppress(redis.exceptions.ConnectionError, AttributeError):
if self.status == self.Status.RUNNING:
Expand Down Expand Up @@ -2416,6 +2417,17 @@ def macho_binaries(self):
def executable_binaries(self):
return self.union(self.win_exes(), self.macho_binaries(), self.elfs())

def with_has_children(self):
"""
Annotate the QuerySet with has_children field based on whether
each resource has any children (subdirectories/files).
"""
children_qs = CodebaseResource.objects.filter(
parent_path=OuterRef("path"),
)

return self.annotate(has_children=Exists(children_qs))


class ScanFieldsModelMixin(models.Model):
"""Fields returned by the ScanCode-toolkit scans."""
Expand Down Expand Up @@ -2739,6 +2751,17 @@ class CodebaseResource(
'Eg.: "/usr/bin/bash" for a path of "tarball-extract/rootfs/usr/bin/bash"'
),
)

parent_path = models.CharField(
max_length=2000,
blank=True,
help_text=_(
"The path of the resource's parent directory. "
"Set to None for top-level (root) resources. "
"Used to efficiently retrieve a directory's contents."
),
)

status = models.CharField(
blank=True,
max_length=50,
Expand Down Expand Up @@ -2832,6 +2855,7 @@ class Meta:
models.Index(fields=["compliance_alert"]),
models.Index(fields=["is_binary"]),
models.Index(fields=["is_text"]),
models.Index(fields=["project", "parent_path"]),
]
constraints = [
models.UniqueConstraint(
Expand All @@ -2844,6 +2868,11 @@ class Meta:
def __str__(self):
return self.path

def save(self, *args, **kwargs):
if self.path and not self.parent_path:
self.parent_path = self.parent_directory() or ""
super().save(*args, **kwargs)

def get_absolute_url(self):
return reverse("resource_detail", args=[self.project.slug, self.path])

Expand Down Expand Up @@ -2914,7 +2943,8 @@ def get_path_segments_with_subpath(self):

def parent_directory(self):
"""Return the parent path for this CodebaseResource or None."""
return parent_directory(self.path, with_trail=False)
parent_path = parent_directory(str(self.path), with_trail=False)
return parent_path or None

def has_parent(self):
"""
Expand Down
29 changes: 29 additions & 0 deletions scanpipe/templates/scanpipe/panels/codebase_tree_panel.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
<ul class="pl-2">
{% for node in children %}
<li class="mb-1">
{% if node.is_dir %}
<div class="tree-node is-flex is-align-items-center has-text-weight-semibold is-clickable px-1" data-folder{% if node.has_children %} data-target="{{ node.path|slugify }}" data-url="{% url 'codebase_resource_tree' slug=project.slug %}?path={{ node.path }}"{% endif %}>
<span class="icon is-small chevron mr-1{% if not node.has_children %} is-invisible{% endif %}" data-chevron>
<i class="fas fa-chevron-right"></i>
</span>
<span class="is-flex is-align-items-center folder-meta">
<span class="icon is-small mr-1">
<i class="fas fa-folder"></i>
</span>
<span>{{ node.name }}</span>
</span>
</div>
{% if node.has_children %}
<div id="dir-{{ node.path|slugify }}" class="ml-4 is-hidden" data-loaded="false"></div>
{% endif %}
{% else %}
<div class="is-flex is-align-items-center ml-5 is-clickable">
<span class="icon is-small mr-1">
<i class="far fa-file"></i>
</span>
<span>{{ node.name }}</span>
</div>
{% endif %}
</li>
{% endfor %}
</ul>
67 changes: 67 additions & 0 deletions scanpipe/templates/scanpipe/resource_tree.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
{% extends "scanpipe/base.html" %}
{% load static humanize %}
{% block title %}ScanCode.io: {{ project.name }} - Resource Tree{% endblock %}

{% block extrahead %}
<style>
.chevron {
transition: transform 0.2s ease;
display: inline-block;
}
.chevron.rotated {
transform: rotate(90deg);
}
</style>
{% endblock %}

{% block content %}
<div id="content-header" class="container is-max-widescreen mb-3">
{% include 'scanpipe/includes/navbar_header.html' %}
<section class="mx-5">
<div class="is-flex is-justify-content-space-between">
{% include 'scanpipe/includes/breadcrumb.html' with linked_project=True current="Resource Tree" %}
</div>
</section>
</div>

<div class="columns is-gapless is-mobile" style="height: 80vh; margin: 0;">
<div class="column is-one-third p-4" style="border-right: 1px solid #ccc; overflow-y: auto;">
<div id="resource-tree">
{% include "scanpipe/panels/codebase_tree_panel.html" with children=children path=path %}
</div>
</div>
<div class="column p-4" style="overflow-y: auto;">
<div id="right-pane">
</div>
</div>
</div>
{% endblock %}

{% block scripts %}
<script>
document.addEventListener("click", async function (e) {
const chevron = e.target.closest("[data-chevron]");
if (chevron) {
const folderNode = chevron.closest("[data-folder]");
const targetId = folderNode.dataset.target;
const url = folderNode.dataset.url;
const icon = chevron.querySelector("i");
const target = document.getElementById("dir-" + targetId);

if (target.dataset.loaded === "true") {
target.classList.toggle("is-hidden");
} else {
target.classList.remove("is-hidden");
const response = await fetch(url + "&tree_panel=true");
target.innerHTML = await response.text();
target.dataset.loaded = "true";
htmx.process(target);
}

chevron.classList.toggle("rotated");
e.stopPropagation();
return;
}
});
</script>
{% endblock %}
18 changes: 18 additions & 0 deletions scanpipe/tests/test_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -3052,6 +3052,24 @@ def test_scanpipe_discovered_package_model_unique_package_uid_in_project(self):
self.assertTrue(package3.package_uid)
self.assertNotEqual(package.package_uid, package3.package_uid)

def test_scanpipe_codebase_resource_queryset_with_has_children(self):
project1 = make_project("Analysis")

make_resource_directory(project1, "parent")
make_resource_file(project1, "parent/child.txt")
make_resource_directory(project1, "empty")

qs = CodebaseResource.objects.filter(project=project1).with_has_children()

resource1 = qs.get(path="parent")
self.assertTrue(resource1.has_children)

resource2 = qs.get(path="parent/child.txt")
self.assertFalse(resource2.has_children)

resource3 = qs.get(path="empty")
self.assertFalse(resource3.has_children)

@skipIf(connection.vendor == "sqlite", "No max_length constraints on SQLite.")
def test_scanpipe_codebase_resource_create_and_add_package_warnings(self):
project1 = make_project("Analysis")
Expand Down
31 changes: 31 additions & 0 deletions scanpipe/tests/test_views.py
Original file line number Diff line number Diff line change
Expand Up @@ -1625,3 +1625,34 @@ def test_project_codebase_resources_export_json(self):

for field in expected_fields:
self.assertIn(field, json_data[0])

def test_scanpipe_views_resource_tree_root_path(self):
make_resource_file(self.project1, path="child1.txt")
make_resource_file(self.project1, path="dir1")

url = reverse("codebase_resource_tree", kwargs={"slug": self.project1.slug})
response = self.client.get(url)
children = response.context["children"]
child1 = children[0]
dir1 = children[1]

self.assertEqual(child1.path, "child1.txt")
self.assertEqual(dir1.path, "dir1")

def test_scanpipe_views_resource_tree_children_path(self):
make_resource_file(self.project1, path="parent/child1.txt")
make_resource_file(self.project1, path="parent/dir1")
make_resource_file(self.project1, path="parent/dir1/child2.txt")

url = reverse("codebase_resource_tree", kwargs={"slug": self.project1.slug})
response = self.client.get(url + "?path=parent&tree_panel=true")
children = response.context["children"]

child1 = children[0]
dir1 = children[1]

self.assertEqual(child1.path, "parent/child1.txt")
self.assertEqual(dir1.path, "parent/dir1")

self.assertFalse(child1.has_children)
self.assertTrue(dir1.has_children)
5 changes: 5 additions & 0 deletions scanpipe/urls.py
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,11 @@
views.ProjectCodebaseView.as_view(),
name="project_codebase",
),
path(
"project/<slug:slug>/codebase_tree/",
views.CodebaseResourceTreeView.as_view(),
name="codebase_resource_tree",
),
path(
"run/<uuid:uuid>/",
views.run_detail_view,
Expand Down
26 changes: 26 additions & 0 deletions scanpipe/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -2742,3 +2742,29 @@ def get_node(self, package):
"children": children,
}
return node


class CodebaseResourceTreeView(ConditionalLoginRequired, generic.DetailView):
template_name = "scanpipe/resource_tree.html"

def get(self, request, *args, **kwargs):
slug = self.kwargs.get("slug")
project = get_object_or_404(Project, slug=slug)
path = request.GET.get("path", "")

children = (
project.codebaseresources.filter(parent_path=path)
.with_has_children()
.only("path", "name", "type")
.order_by("path")
)

context = {
"project": project,
"path": path,
"children": children,
}

if request.GET.get("tree_panel") == "true":
return render(request, "scanpipe/panels/codebase_tree_panel.html", context)
return render(request, self.template_name, context)