Skip to content

Rust: Use QL computed canonical paths in MaD Field tokens #19667

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 45 additions & 47 deletions misc/scripts/models-as-data/bulk_generate_mad.py
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ def build_database(
return database_dir


def generate_models(config, project: Project, database_dir: str) -> None:
def generate_models(config, args, project: Project, database_dir: str) -> None:
"""
Generate models for a project.

Expand All @@ -243,6 +243,8 @@ def generate_models(config, project: Project, database_dir: str) -> None:
generator.generateSources = should_generate_sources(project)
generator.generateSummaries = should_generate_summaries(project)
generator.setenvironment(database=database_dir, folder=name)
generator.threads = args.codeql_threads
generator.ram = args.codeql_ram
generator.run()


Expand Down Expand Up @@ -333,43 +335,44 @@ def pretty_name_from_artifact_name(artifact_name: str) -> str:

def download_dca_databases(
language: str,
experiment_name: str,
experiment_names: list[str],
pat: str,
projects: List[Project],
) -> List[tuple[Project, str | None]]:
"""
Download databases from a DCA experiment.
Args:
experiment_name: The name of the DCA experiment to download databases from.
experiment_names: The names of the DCA experiments to download databases from.
pat: Personal Access Token for GitHub API authentication.
projects: List of projects to download databases for.
Returns:
List of (project_name, database_dir) pairs, where database_dir is None if the download failed.
"""
print("\n=== Finding projects ===")
response = get_json_from_github(
f"https://raw.githubusercontent.com/github/codeql-dca-main/data/{experiment_name}/reports/downloads.json",
pat,
)
targets = response["targets"]
project_map = {project["name"]: project for project in projects}
analyzed_databases = {}
for data in targets.values():
downloads = data["downloads"]
analyzed_database = downloads["analyzed_database"]
artifact_name = analyzed_database["artifact_name"]
pretty_name = pretty_name_from_artifact_name(artifact_name)

if not pretty_name in project_map:
print(f"Skipping {pretty_name} as it is not in the list of projects")
continue

if pretty_name in analyzed_databases:
print(
f"Skipping previous database {analyzed_databases[pretty_name]['artifact_name']} for {pretty_name}"
)
for experiment_name in experiment_names:
response = get_json_from_github(
f"https://raw.githubusercontent.com/github/codeql-dca-main/data/{experiment_name}/reports/downloads.json",
pat,
)
targets = response["targets"]
for data in targets.values():
downloads = data["downloads"]
analyzed_database = downloads["analyzed_database"]
artifact_name = analyzed_database["artifact_name"]
pretty_name = pretty_name_from_artifact_name(artifact_name)

if not pretty_name in project_map:
print(f"Skipping {pretty_name} as it is not in the list of projects")
continue

if pretty_name in analyzed_databases:
print(
f"Skipping previous database {analyzed_databases[pretty_name]['artifact_name']} for {pretty_name}"
)

analyzed_databases[pretty_name] = analyzed_database
analyzed_databases[pretty_name] = analyzed_database

def download_and_decompress(analyzed_database: dict) -> str:
artifact_name = analyzed_database["artifact_name"]
Expand Down Expand Up @@ -450,23 +453,6 @@ def main(config, args) -> None:
if not os.path.exists(build_dir):
os.makedirs(build_dir)

# Check if any of the MaD directories contain working directory changes in git
for project in projects:
mad_dir = get_mad_destination_for_project(config, project["name"])
if os.path.exists(mad_dir):
git_status_output = subprocess.check_output(
["git", "status", "-s", mad_dir], text=True
).strip()
if git_status_output:
print(
f"""ERROR: Working directory changes detected in {mad_dir}.

Before generating new models, the existing models are deleted.

To avoid loss of data, please commit your changes."""
)
sys.exit(1)

database_results = []
match get_strategy(config):
case "repo":
Expand All @@ -477,8 +463,8 @@ def main(config, args) -> None:
projects,
)
case "dca":
experiment_name = args.dca
if experiment_name is None:
experiment_names = args.dca
if experiment_names is None:
print("ERROR: --dca argument is required for DCA strategy")
sys.exit(1)

Expand All @@ -492,7 +478,7 @@ def main(config, args) -> None:
pat = f.read().strip()
database_results = download_dca_databases(
language,
experiment_name,
experiment_names,
pat,
projects,
)
Expand All @@ -518,7 +504,7 @@ def main(config, args) -> None:

for project, database_dir in database_results:
if database_dir is not None:
generate_models(config, project, database_dir)
generate_models(config, args, project, database_dir)


if __name__ == "__main__":
Expand All @@ -529,14 +515,26 @@ def main(config, args) -> None:
parser.add_argument(
"--dca",
type=str,
help="Name of a DCA run that built all the projects",
required=False,
help="Name of a DCA run that built all the projects. Can be repeated, with sources taken from all provided runs, "
"the last provided ones having priority",
action="append",
)
parser.add_argument(
"--pat",
type=str,
help="Path to a file containing the PAT token required to grab DCA databases (the same as the one you use for DCA)",
required=False,
)
parser.add_argument(
"--codeql-ram",
type=int,
help="What `--ram` value to pass to `codeql` while generating models (by default the flag is not passed)",
default=None,
)
parser.add_argument(
"--codeql-threads",
type=int,
help="What `--threads` value to pass to `codeql` (default %(default)s)",
default=0,
)
args = parser.parse_args()

Expand Down
12 changes: 9 additions & 3 deletions misc/scripts/models-as-data/generate_mad.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ def __init__(self, language):
self.generateTypeBasedSummaries = False
self.dryRun = False
self.dirname = "modelgenerator"
self.ram = 2**15
self.threads = 8


def setenvironment(self, database, folder):
Expand Down Expand Up @@ -138,8 +140,12 @@ def runQuery(self, query):
queryFile = os.path.join(self.codeQlRoot, f"{self.language}/ql/src/utils/{self.dirname}", query)
resultBqrs = os.path.join(self.workDir, "out.bqrs")

helpers.run_cmd(['codeql', 'query', 'run', queryFile, '--database',
self.database, '--output', resultBqrs, '--threads', '8', '--ram', '32768'], "Failed to generate " + query)
cmd = ['codeql', 'query', 'run', queryFile, '--database', self.database, '--output', resultBqrs]
if self.threads is not None:
cmd += ["--threads", str(self.threads)]
if self.ram is not None:
cmd += ["--ram", str(self.ram)]
helpers.run_cmd(cmd, "Failed to generate " + query)

return helpers.readData(self.workDir, resultBqrs)

Expand Down Expand Up @@ -214,4 +220,4 @@ def run(self):
self.save(typeBasedContent, ".typebased.model.yml")

if __name__ == '__main__':
Generator.make().run()
Generator.make().run()
19 changes: 2 additions & 17 deletions rust/ql/lib/codeql/rust/dataflow/internal/FlowSummaryImpl.qll
Original file line number Diff line number Diff line change
Expand Up @@ -68,29 +68,14 @@ module Input implements InputSig<Location, RustDataFlow> {
result = "Field" and
(
exists(Addressable a, int pos, string prefix |
// TODO: calculate in QL
arg = prefix + "(" + pos + ")" and
(
prefix = a.getExtendedCanonicalPath()
or
a = any(OptionEnum o).getSome() and
prefix = "crate::option::Option::Some"
or
exists(string name |
a = any(ResultEnum r).getVariant(name) and
prefix = "crate::result::Result::" + name
)
)
arg = prefix + "(" + pos + ")" and prefix = a.getCanonicalPath()
|
c.(TupleFieldContent).isStructField(a, pos)
or
c.(TupleFieldContent).isVariantField(a, pos)
)
or
exists(Addressable a, string field |
// TODO: calculate in QL
arg = a.getExtendedCanonicalPath() + "::" + field
|
exists(Addressable a, string field | arg = a.getCanonicalPath() + "::" + field |
c.(StructFieldContent).isStructField(a, field)
or
c.(StructFieldContent).isVariantField(a, field)
Expand Down
4 changes: 2 additions & 2 deletions rust/ql/lib/codeql/rust/dataflow/internal/ModelsAsData.qll
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,9 @@
* - `ReturnValue`: the value returned by a function call.
* - `Element`: an element in a collection.
* - `Field[t::f]`: field `f` of the variant/struct with canonical path `t`, for example
* `Field[crate::ihex::Record::Data::value]`.
* `Field[ihex::Record::Data::value]`.
* - `Field[t(i)]`: position `i` inside the variant/struct with canonical path `v`, for example
* `Field[crate::option::Option::Some(0)]`.
* `Field[core::option::Option::Some(0)]`.
* - `Field[i]`: the `i`th element of a tuple.
* 4. The `kind` column is a tag that can be referenced from QL to determine to
* which classes the interpreted elements should be added. For example, for
Expand Down
2 changes: 1 addition & 1 deletion rust/ql/lib/codeql/rust/frameworks/async-rs.model.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@ extensions:
pack: codeql/rust-all
extensible: sourceModel
data:
- ["repo:https://github.com/async-rs/async-std:async-std", "<crate::net::tcp::stream::TcpStream>::connect", "ReturnValue.Future.Field[crate::result::Result::Ok(0)]", "remote", "manual"]
- ["repo:https://github.com/async-rs/async-std:async-std", "<crate::net::tcp::stream::TcpStream>::connect", "ReturnValue.Future.Field[core::result::Result::Ok(0)]", "remote", "manual"]
6 changes: 3 additions & 3 deletions rust/ql/lib/codeql/rust/frameworks/futures.model.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ extensions:
- ["repo:https://github.com/rust-lang/futures-rs:futures-util", "crate::io::AsyncBufReadExt::read_line", "Argument[self].Reference", "Argument[0].Reference", "taint", "manual"]
- ["repo:https://github.com/rust-lang/futures-rs:futures-util", "crate::io::AsyncBufReadExt::read_until", "Argument[self]", "Argument[1].Reference", "taint", "manual"]
- ["repo:https://github.com/rust-lang/futures-rs:futures-util", "crate::io::AsyncBufReadExt::read_until", "Argument[self].Reference", "Argument[1].Reference", "taint", "manual"]
- ["repo:https://github.com/rust-lang/futures-rs:futures-util", "crate::io::AsyncBufReadExt::fill_buf", "Argument[self]", "ReturnValue.Future.Field[crate::result::Result::Ok(0)]", "taint", "manual"]
- ["repo:https://github.com/rust-lang/futures-rs:futures-util", "crate::io::AsyncBufReadExt::fill_buf", "Argument[self]", "ReturnValue.Future.Field[core::result::Result::Ok(0)]", "taint", "manual"]
- ["repo:https://github.com/rust-lang/futures-rs:futures-util", "crate::io::AsyncBufReadExt::lines", "Argument[self]", "ReturnValue", "taint", "manual"]
- ["repo:https://github.com/rust-lang/futures-rs:futures-util", "crate::stream::stream::StreamExt::next", "Argument[self]", "ReturnValue.Future.Field[crate::option::Option::Some(0)]", "taint", "manual"]
- ["repo:https://github.com/rust-lang/futures-rs:futures-util", "<crate::io::buf_reader::BufReader as crate::if_std::AsyncBufRead>::poll_fill_buf", "Argument[self].Reference", "ReturnValue.Field[crate::task::poll::Poll::Ready(0)].Field[crate::result::Result::Ok(0)]", "taint", "manual"]
- ["repo:https://github.com/rust-lang/futures-rs:futures-util", "crate::stream::stream::StreamExt::next", "Argument[self]", "ReturnValue.Future.Field[core::option::Option::Some(0)]", "taint", "manual"]
- ["repo:https://github.com/rust-lang/futures-rs:futures-util", "<crate::io::buf_reader::BufReader as crate::if_std::AsyncBufRead>::poll_fill_buf", "Argument[self].Reference", "ReturnValue.Field[core::task::poll::Poll::Ready(0)].Field[core::result::Result::Ok(0)]", "taint", "manual"]
8 changes: 4 additions & 4 deletions rust/ql/lib/codeql/rust/frameworks/http.model.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@ extensions:
pack: codeql/rust-all
extensible: sourceModel
data:
- ["repo:https://github.com/hyperium/hyper:hyper", "<crate::client::conn::http1::SendRequest>::send_request", "ReturnValue.Future.Field[crate::result::Result::Ok(0)]", "remote", "manual"]
- ["repo:https://github.com/hyperium/hyper:hyper", "<crate::client::conn::http2::SendRequest>::send_request", "ReturnValue.Future.Field[crate::result::Result::Ok(0)]", "remote", "manual"]
- ["repo:https://github.com/hyperium/hyper:hyper", "<crate::client::conn::http1::SendRequest>::try_send_request", "ReturnValue.Future.Field[crate::result::Result::Ok(0)]", "remote", "manual"]
- ["repo:https://github.com/hyperium/hyper:hyper", "<crate::client::conn::http2::SendRequest>::try_send_request", "ReturnValue.Future.Field[crate::result::Result::Ok(0)]", "remote", "manual"]
- ["repo:https://github.com/hyperium/hyper:hyper", "<crate::client::conn::http1::SendRequest>::send_request", "ReturnValue.Future.Field[core::result::Result::Ok(0)]", "remote", "manual"]
- ["repo:https://github.com/hyperium/hyper:hyper", "<crate::client::conn::http2::SendRequest>::send_request", "ReturnValue.Future.Field[core::result::Result::Ok(0)]", "remote", "manual"]
- ["repo:https://github.com/hyperium/hyper:hyper", "<crate::client::conn::http1::SendRequest>::try_send_request", "ReturnValue.Future.Field[core::result::Result::Ok(0)]", "remote", "manual"]
- ["repo:https://github.com/hyperium/hyper:hyper", "<crate::client::conn::http2::SendRequest>::try_send_request", "ReturnValue.Future.::Result::Ok(0)]", "remote", "manual"]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This one looks broken, we've lost Field[core::result

- ["repo:https://github.com/hyperium/hyper:hyper", "<crate::client::Client>::get", "ReturnValue.Future", "remote", "manual"]
- ["repo:https://github.com/hyperium/hyper:hyper", "<crate::client::Client>::request", "ReturnValue.Future", "remote", "manual"]
- ["repo:https://github.com/hyperium/hyper-util:hyper-util", "<crate::client::legacy::Client>::get", "ReturnValue.Future", "remote", "manual"]
Expand Down
6 changes: 3 additions & 3 deletions rust/ql/lib/codeql/rust/frameworks/log.model.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@ extensions:
- ["lang:std", "<crate::io::stdio::StderrLock as crate::io::Write>::write", "Argument[0]", "log-injection", "manual"]
- ["lang:std", "<crate::io::stdio::StderrLock as crate::io::Write>::write_all", "Argument[0]", "log-injection", "manual"]
- ["lang:core", "crate::panicking::panic_fmt", "Argument[0]", "log-injection", "manual"]
- ["lang:core", "crate::panicking::assert_failed", "Argument[3].Field[crate::option::Option::Some(0)]", "log-injection", "manual"]
- ["lang:core", "crate::panicking::assert_failed", "Argument[3].Field[core::option::Option::Some(0)]", "log-injection", "manual"]
- ["lang:core", "<crate::option::Option>::expect", "Argument[0]", "log-injection", "manual"]
- ["repo:https://github.com/DesmondWillowbrook/rs-log_err:log_err", "<crate::option::Option as crate::LogErrOption>::log_expect", "Argument[0]", "log-injection", "manual"]
- ["repo:https://github.com/DesmondWillowbrook/rs-log_err:log_err", "<crate::result::Result as crate::LogErrResult>::log_unwrap", "Argument[self].Field[crate::result::Result::Err(0)]", "log-injection", "manual"]
- ["repo:https://github.com/DesmondWillowbrook/rs-log_err:log_err", "<crate::result::Result as crate::LogErrResult>::log_unwrap", "Argument[self].Field[core::result::Result::Err(0)]", "log-injection", "manual"]
- ["repo:https://github.com/DesmondWillowbrook/rs-log_err:log_err", "<crate::result::Result as crate::LogErrResult>::log_expect", "Argument[0]", "log-injection", "manual"]
- ["repo:https://github.com/DesmondWillowbrook/rs-log_err:log_err", "<crate::result::Result as crate::LogErrResult>::log_expect", "Argument[self].Field[crate::result::Result::Err(0)]", "log-injection", "manual"]
- ["repo:https://github.com/DesmondWillowbrook/rs-log_err:log_err", "<crate::result::Result as crate::LogErrResult>::log_expect", "Argument[self].Field[core::result::Result::Err(0)]", "log-injection", "manual"]
Loading