Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add dependencies to rpm #4084

Open
wants to merge 1 commit into
base: develop
Choose a base branch
from
+251 −11
Open

Add dependencies to rpm #4084

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
188 changes: 182 additions & 6 deletions src/packagedcode/rpm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from collections import namedtuple
from pathlib import Path

from packageurl import PackageURL
from packagedcode import models
from packagedcode import nevra
from packagedcode.licensing import RESOURCE_TO_PACKAGE_LICENSE_FIELDS
Expand Down Expand Up @@ -44,7 +45,6 @@ def logger_debug(*args):
"""
Support for RPMs, installed databases and spec files.
"""
# TODO: retrieve dependencies

# TODO: parse spec files see:
# http://www.faqs.org/docs/artu/ch05s02.html#id2906931%29.)
Expand Down Expand Up @@ -125,7 +125,6 @@ def to_string(self):
return vr


# TODO: add dependencies!!!
class BaseRpmInstalledDatabaseHandler(models.DatafileHandler):

@classmethod
Expand All @@ -135,13 +134,45 @@ def parse(cls, location, package_only=False):
loc_path = Path(location)
rpmdb_loc = str(loc_path.parent)

rpm_tags = get_rpm_tags(location, include_desc=True)
alok1304 marked this conversation as resolved.
Show resolved Hide resolved

if TRACE: logger_debug('recognize: rpm_tags', rpm_tags)
if not rpm_tags:
return

dependencies = []
name = rpm_tags.name
version = rpm_tags.version
is_pinned = version is not None and version != ""

# Construct PackageURL without '@version' if version is missing
purl = PackageURL(
type="rpm",
namespace=None, # RPMs typically don't use namespaces
name=name,
version=version if is_pinned else None
)

# Prepare the dependent package model
dependencies.append(
models.DependentPackage(
purl=purl.to_string(),
scope="dependencies",
is_runtime=True,
is_optional=False,
is_pinned=is_pinned,
extracted_requirement=version,
)
)

# dump and parse the rpmdb to XMLish
xmlish_loc = collect_installed_rpmdb_xmlish_from_rpmdb_loc(rpmdb_loc=rpmdb_loc)
package_data = parse_rpm_xmlish(
location=xmlish_loc,
datasource_id=cls.datasource_id,
package_type=cls.default_package_type,
package_only=package_only,
dependencies=dependencies,
)
# TODO: package_data.namespace = cls.default_package_namespace
return package_data
Expand Down Expand Up @@ -225,7 +256,6 @@ def assemble(cls, package_data, resource, codebase, package_adder):
yield resource


# TODO: add dependencies!!!
class RpmInstalledNdbDatabaseHandler(BaseRpmInstalledDatabaseHandler):
# used by recent Suse
datasource_id = 'rpm_installed_database_ndb'
Expand All @@ -237,8 +267,48 @@ class RpmInstalledNdbDatabaseHandler(BaseRpmInstalledDatabaseHandler):
description = 'RPM installed package NDB database'
documentation_url = 'https://fedoraproject.org/wiki/Changes/NewRpmDBFormat'

@classmethod
def parse(cls, location, package_only=False):
rpm_tags = get_rpm_tags(location, include_desc=True)

if TRACE: logger_debug('recognize: rpm_tags', rpm_tags)
if not rpm_tags:
return

dependencies = []
name = rpm_tags.name
version = rpm_tags.version
is_pinned = version is not None and version != ""
alok1304 marked this conversation as resolved.
Show resolved Hide resolved

# Construct PackageURL without '@version' if version is missing
purl = PackageURL(
type="rpm",
namespace=None, # RPMs typically don't use namespaces
alok1304 marked this conversation as resolved.
Show resolved Hide resolved
name=name,
version=version if is_pinned else None
)

# Prepare the dependent package model
alok1304 marked this conversation as resolved.
Show resolved Hide resolved
dependencies.append(
alok1304 marked this conversation as resolved.
Show resolved Hide resolved
models.DependentPackage(
purl=purl.to_string(),
scope="dependencies",
is_runtime=True,
is_optional=False,
is_pinned=is_pinned,
extracted_requirement=version,
)
)

package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
dependencies=dependencies,
)

yield models.PackageData.from_data(package_data, package_only)


# TODO: add dependencies!!!
class RpmInstalledSqliteDatabaseHandler(BaseRpmInstalledDatabaseHandler):
# used by newer RHEL/CentOS/Fedora/CoreOS
# Filetype: SQLite 3.x database, ...
Expand All @@ -253,8 +323,48 @@ class RpmInstalledSqliteDatabaseHandler(BaseRpmInstalledDatabaseHandler):
description = 'RPM installed package SQLite database'
documentation_url = 'https://fedoraproject.org/wiki/Changes/Sqlite_Rpmdb'

@classmethod
def parse(cls, location, package_only=False):
rpm_tags = get_rpm_tags(location, include_desc=True)
alok1304 marked this conversation as resolved.
Show resolved Hide resolved

if TRACE: logger_debug('recognize: rpm_tags', rpm_tags)
if not rpm_tags:
return

dependencies = []
name = rpm_tags.name
version = rpm_tags.version
is_pinned = version is not None and version != ""

# Construct PackageURL without '@version' if version is missing
purl = PackageURL(
type="rpm",
namespace=None, # RPMs typically don't use namespaces
name=name,
version=version if is_pinned else None
)

# Prepare the dependent package model
dependencies.append(
models.DependentPackage(
purl=purl.to_string(),
scope="dependencies",
is_runtime=True,
is_optional=False,
is_pinned=is_pinned,
extracted_requirement=version,
)
)

package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
dependencies=dependencies,
)

yield models.PackageData.from_data(package_data, package_only)


# TODO: add dependencies!!!
class RpmInstalledBdbDatabaseHandler(BaseRpmInstalledDatabaseHandler):
# used by legacy RHEL/CentOS/Fedora/Suse
datasource_id = 'rpm_installed_database_bdb'
Expand All @@ -267,6 +377,47 @@ class RpmInstalledBdbDatabaseHandler(BaseRpmInstalledDatabaseHandler):
description = 'RPM installed package BDB database'
documentation_url = 'https://man7.org/linux/man-pages/man8/rpmdb.8.html'

@classmethod
def parse(cls, location, package_only=False):
alok1304 marked this conversation as resolved.
Show resolved Hide resolved
rpm_tags = get_rpm_tags(location, include_desc=True)

if TRACE: logger_debug('recognize: rpm_tags', rpm_tags)
if not rpm_tags:
return

dependencies = []
name = rpm_tags.name
version = rpm_tags.version
is_pinned = version is not None and version != ""

# Construct PackageURL without '@version' if version is missing
purl = PackageURL(
type="rpm",
namespace=None, # RPMs typically don't use namespaces
name=name,
version=version if is_pinned else None
)

# Prepare the dependent package model
dependencies.append(
models.DependentPackage(
purl=purl.to_string(),
scope="dependencies",
is_runtime=True,
is_optional=False,
is_pinned=is_pinned,
extracted_requirement=version,
)
)

package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
dependencies=dependencies,
)

yield models.PackageData.from_data(package_data, package_only)


# TODO: implement me!!@
class RpmSpecfileHandler(models.NonAssemblableDatafileHandler):
Expand All @@ -278,7 +429,6 @@ class RpmSpecfileHandler(models.NonAssemblableDatafileHandler):
documentation_url = 'https://en.wikipedia.org/wiki/RPM_Package_Manager'


# TODO: add dependencies!!!
class RpmArchiveHandler(models.DatafileHandler):
datasource_id = 'rpm_archive'
path_patterns = ('*.rpm', '*.src.rpm', '*.srpm', '*.mvl', '*.vip',)
Expand Down Expand Up @@ -354,6 +504,30 @@ def parse(cls, location, package_only=False):

description = build_description(summary=rpm_tags.summary, description=rpm_tags.description)

dependencies = []
name = rpm_tags.name
version = rpm_tags.version
is_pinned = version is not None and version != ""

# Construct PackageURL without '@version' if version is missing
purl = PackageURL(
type="rpm",
namespace=None, # RPMs typically don't use namespaces
name=name,
version=version if is_pinned else None
)

# Prepare the dependent package model
dependencies.append(
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since when there is a single dependency?

Copy link
Author

@alok1304 alok1304 Jan 11, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@pombredanne Can you suggest how I proceed to find multiple dependencies, I tried by Adding RPMTAG_REQUIRES and RPMTAG_REQUIREVERSION in RPMtags in pyrpm.py but in RPMTAG_REQUIRES I got this like eg: ['/bin/sh', '/bin/sh', '/bin/sh', '/bin/sh', '/bin/sh', 'rpmlib(PayloadFilesHavePrefix)', 'rpmlib(CompressedFileNames)', 'rpmlib(PayloadIsBzip2)']
corresponding I got require_version=[None, None, None, None, None, '4.0-1', '3.0.4-1', '3.0.5-1']
source: http://ftp.rpm.org/max-rpm/ch-queryformat-tags.html (For rpm tages)

can you tell how I proceed to find out their package name and their version, in rmp_requires, these represent capabilities or libraries, not actual packages

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since when there is a single dependency?

I do for single dependency only.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I do for single dependency only.

This cannot work. We need all of them.

  1. You can use "get_rpm_tags" only for a .rpm file, but then you do not need to call it because this is already called
  2. for installed RPM databases, you can use exclusively the data in parse_rpm_xmlish https://github.com/aboutcode-org/scancode-toolkit/pull/4084/files#diff-17fc898047e57c63a40599213e50fb03f8d853b60ea7790133ace1c6ab2b9709R170 and if it does not return what you need, parse_rpm_xmlish needs to be updated in https://github.com/aboutcode-org/scancode-toolkit/blob/develop/src/packagedcode/rpm_installed.py. Look at require*, provide* and conflict* for instance in https://github.com/aboutcode-org/scancode-toolkit/edit/develop/tests/packagedcode/data/rpm_installed/distro-xmlish/rhel-rpms.xmlish

models.DependentPackage(
purl=purl.to_string(),
scope="dependencies",
is_runtime=True,
is_optional=False,
is_pinned=is_pinned,
extracted_requirement=version,
)
)
if TRACE:
data = dict(
name=name,
Expand All @@ -363,6 +537,7 @@ def parse(cls, location, package_only=False):
parties=parties,
extracted_license_statement=rpm_tags.license or None,
source_packages=source_packages,
dependencies=dependencies,
)
logger_debug('recognize: data to create a package:\n', data)

Expand All @@ -377,6 +552,7 @@ def parse(cls, location, package_only=False):
parties=parties,
extracted_license_statement=rpm_tags.license or None,
source_packages=source_packages,
dependencies=dependencies,
)

if TRACE:
Expand Down
14 changes: 13 additions & 1 deletion tests/packagedcode/data/plugin/rpm-package-expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,19 @@
"is_private": false,
"is_virtual": false,
"extra_data": {},
"dependencies": [],
"dependencies": [
{
"purl": "pkg:rpm/[email protected]",
"extracted_requirement": "2.0",
"scope": "dependencies",
"is_runtime": true,
"is_optional": false,
"is_pinned": true,
"is_direct": true,
"resolved_package": {},
"extra_data": {}
}
],
"repository_homepage_url": null,
"repository_download_url": null,
"api_data_url": null,
Expand Down
14 changes: 13 additions & 1 deletion .../packagedcode/data/rpm/header/libproxy-bin-0.3.0-4.el6_3.x86_64.rpm-package-expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,19 @@
"is_private": false,
"is_virtual": false,
"extra_data": {},
"dependencies": [],
"dependencies": [
{
"purl": "pkg:rpm/[email protected]",
"extracted_requirement": "0.3.0",
"scope": "dependencies",
"is_runtime": true,
"is_optional": false,
"is_pinned": true,
"is_direct": true,
"resolved_package": {},
"extra_data": {}
}
],
"repository_homepage_url": null,
"repository_download_url": null,
"api_data_url": null,
Expand Down
14 changes: 13 additions & 1 deletion ...agedcode/data/rpm/header/libproxy-bin-0.3.0-4.el6_3.x86_64.rpm-package-only-expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,19 @@
"is_private": false,
"is_virtual": false,
"extra_data": {},
"dependencies": [],
"dependencies": [
{
"purl": "pkg:rpm/[email protected]",
"extracted_requirement": "0.3.0",
"scope": "dependencies",
"is_runtime": true,
"is_optional": false,
"is_pinned": true,
"is_direct": true,
"resolved_package": {},
"extra_data": {}
}
],
"repository_homepage_url": null,
"repository_download_url": null,
"api_data_url": null,
Expand Down
32 changes: 30 additions & 2 deletions tests/scancode/data/rpm/fping-2.4-0.b2.rhfc1.dag.i386.rpm.expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,23 @@
"purl": "pkg:rpm/[email protected]"
}
],
"dependencies": [],
"dependencies": [
{
"purl": "pkg:rpm/[email protected]",
"extracted_requirement": "2.4",
"scope": "dependencies",
"is_runtime": true,
"is_optional": false,
"is_pinned": true,
"is_direct": true,
"resolved_package": {},
"extra_data": {},
"dependency_uid": "pkg:rpm/[email protected]?uuid=4c055d32-6a8c-465e-9c76-f2714b7ce23f",
"for_package_uid": "pkg:rpm/[email protected]?uuid=1903aca6-54da-49d6-a219-ff432eff4398",
"datafile_path": "fping-2.4-0.b2.rhfc1.dag.i386.rpm",
"datasource_id": "rpm_archive"
}
],
"files": [
{
"path": "fping-2.4-0.b2.rhfc1.dag.i386.rpm",
Expand Down Expand Up @@ -158,7 +174,19 @@
"is_private": false,
"is_virtual": false,
"extra_data": {},
"dependencies": [],
"dependencies": [
{
"purl": "pkg:rpm/[email protected]",
"extracted_requirement": "2.4",
"scope": "dependencies",
"is_runtime": true,
"is_optional": false,
"is_pinned": true,
"is_direct": true,
"resolved_package": {},
"extra_data": {}
}
],
"repository_homepage_url": null,
"repository_download_url": null,
"api_data_url": null,
Expand Down
Loading