diff --git a/giza/README.rst b/giza/README.rst index 1b9eba0dc..dbce8360f 100644 --- a/giza/README.rst +++ b/giza/README.rst @@ -9,40 +9,15 @@ documentation project; however, its design is sufficiently generic to be able to facilitate the builds of multiple documentation resources produced at MongoDB. -Features and Goals ------------------- +Resources +--------- -Giza has the following objectives and primary features: +`Giza on PyPi `_ -- Facilitate fully-local test builds. Contributors to the - documentation should be able to generate the documentation using the - exact same process used to produce the production version of the - resources. +`Giza on Github `_ -- Generate content from structured forms into reStructuredText that - Sphinx can publish. In an effort to manage duplicated content, and - facilitate sustainable content reuse, Giza translates structured - content, procedures, including command line interfaces, tables of - contents, and API interfaces. - -- Run multiple Sphinx builds concurrently. Practically speaking, - building the documentation requires running Sphinx multiple - times. Building the documentation requires multiple invocations of - Sphinx, to produce: - - - multiple versions of the manual (i.e. HTML, ``json``, PDF, ePub, - etc.) - - - transitions of the content in different human languages. - - - different editions of a text to address different editions of a - single resource. (e.g. a student and instructor version of a - training resource.) - - Internally, Sphinx itself is not optimally parallelized, and - it's considerably more efficient to run multiple Sphinx processes in - parallel, particularly for larger resources and as the matrix of - required build artifacts grows. +File issues in the `MongoDB DOCS Jira Project +`_. Installation ------------ @@ -66,212 +41,10 @@ At any time, you can install the latest version with the following pip install giza To install the optional ``github`` and ``jira`` integration, use the -following command: +following command: :: pip install giza [jira,github] -Use ---- - -Make Interface -~~~~~~~~~~~~~~ - -Giza is fully integrated into the ``Makefile`` system present in all -MongoDB documentation repositories. Typically you will run builds -using ``make html``, for the html output, or ``make latex`` for the -PDF build. There are two major important targets: ``publish`` that -builds the full production build locally, and ``push`` that is -equivalent to ``publish`` but also uploads all artifacts the resource -to the production web servers. - -During the transition to Giza from the legacy system, all Giza -targets have ``giza-`` prefixes, so to use Giza targets you would use -``make giza-html`` and ``make giza-publish``. - -Direct Use -~~~~~~~~~~ - -While most common operations are wrapped in familiar ``make`` targets, -you can run Giza directly from the command line. This section provides -an overview of these operations. See the output of ``-help`` at all -levels for specific syntax. - -Sphinx -`````` - -The following commands will build the ``html`` version of the -resource: :: - - giza sphinx --builder html - giza sphinx -b html - -Replace ``html`` with any Sphinx builder you wish to use. You can -specify a list, space separated, of builders to build multiple formats -at once. For example: :: - - giza sphinx --builder latex dirhtml json html singlehtml epub man - -For the MongoDB Manual this is equivalent to the ``make publish`` -operation. For projects that have multiple editions, you can specify -the edition as a section option, for example: :: - - giza sphinx --builder latex json html dirhtml singlehtml --edition saas hosted - -This is the Giza command to build the full MMS documentation, and it -builds ``saas`` and ``hosted`` versions of the manual for 5 sphinx -output formats. - -Diagnostics -``````````` - -You can use the ``giza config`` command to see a rendered version of -the configuration object used during builds. ``config`` allows you to -see how specifying a language or edition will affect the config -object. For example: :: - - giza config - giza config --edition saas - giza config --edition hosted - giza config --edition saas --language es - giza config --edition hosted --language fr - giza config -e saas -l es - giza config -e hosted -l fr - -Deploying -````````` - -There are two targets that deploy built documentation to the -production environment: ``deploy``, which only uploads the -resources; and ``push``, which builds and then deploys the -resources. - -Each branch and repository defines its behavior of the deployment in a -*push* config file. These define a number of "push targets" that -describe how to upload the artifacts. When you run a deploy operation, you -specify one or more of these push targets and Giza will deploy the -artifacts specified in the configuration. - -``push`` takes arguments that are the combination of the ``sphinx`` -command and the ``deploy`` command. Consider the following commands: - - giza deploy --target push - giza deploy --target stage - giza deploy --target push-saas - giza deploy --target stage-saas - giza push --deploy push-saas push-hosted --builder latex json html dirhtml singlehtml --edition saas hosted --language es - giza push --deploy push-saas push-hosted --builder latex json html dirhtml singlehtml --edition saas hosted - -Add the ``--dry-run`` or ``-d`` option to any ``deploy`` command to -avoid actually uploading artifacts during testing. - -Git -``` - -Giza provides wrappers for several common ``git`` operations. You can -use Giza to apply the patch from a github pull request or from a -single Github commit: :: - - giza git am -p - giza git am --patch - -Replace ```` with the ID of a pull request against the -repository that repository. You can apply any object from github, by -passing a full github URI as the ````. - -All ``giza git`` commands support a ``--branch`` argument that allows -them to perform their operation on a different branch. For example: :: - - giza git am --patch 1066 --branch v4.2 - giza git am -p 1066 -b v4.2 - -You can also cherry-pick commits from the local repository onto the -current branch: :: - - giza git cp --commits a5b8087 - giza git cp -c a5b8087 - -The ``git cp`` command allows you to cherry pick a list of commits, -but is most useful in combination with the ``--branch`` option to -apply commits to other branches, as in the following examples: :: - - giza git cp --commits a5b8087 8f9150a 2eb441b - giza git cp -c a5b8087 8f9150a 2eb441b - - giza git cp --commits a5b8087 8f9150a 2eb441b --branch v0.2 - giza git cp -c a5b8087 8f9150a 2eb441b --branch v0.2 - -Additional Giza Operations -`````````````````````````` - -``generate`` -'''''''''''' - -These operations generate content or fetch data used by the build -without generating the full artifacts. Useful for debugging and -testing. In normal operations the ``sphinx`` operations generate -require inputs, and these operations are not needed. - -``generate`` provides the following operations to generate content. - -- ``api`` -- ``assets`` -- ``images`` -- ``intersphinx`` -- ``redirects`` -- ``options`` -- ``primer`` -- ``steps`` -- ``tables`` -- ``toc`` - -``includes`` -'''''''''''' - -The ``includes`` operations introspect the resources' content reuse, -and allow writers to be able to see the dependency relationship -between source files. ``giza includes`` has the following additional -operations: - -- ``recursive``: returns a list of all files that also include other - files. - -- ``changes``: returns a list of all files in the repository affected - indirectly by uncommitted changes to the current repository. (Requires - `PyGit2 `_) - -- ``once``: returns a list of all included files that are only used - once. - -- ``unused``: returns a list of all included files that are not used - at all. - -- ``graph``: return a document that maps include files to the - dependent source files. Includes the ``--filter`` option, which - allows you to specify a prefix of included files to limit the size - or scope of the graph. - -``package`` -''''''''''' - -Giza provides support for creating "packages" of build artifacts that -you can use to deploy a version of the resource produced on a -different system or at a different time. This makes it possible to -safely deploy a sequence of builds in quick succession. The -``package`` command provides the following options: - -- ``create``: Given a *push target*, build ha package of the current - build output and the current configuration object. - -- ``fetch``: Given a URL, download the package to the - local "build archive." Will refuse to download a package that - already exists in the build archive. - -- ``unwind``: Given a path or URL of a package, extract the package to - the "public output" directory used for staging. - -- ``deploy``: Given a *push target* and the path or URL of a package, - extract the package and upload those artifacts. - Additional Components --------------------- diff --git a/giza/docs/source/core/giza.txt b/giza/docs/source/core/giza.txt new file mode 100644 index 000000000..a230a36af --- /dev/null +++ b/giza/docs/source/core/giza.txt @@ -0,0 +1,57 @@ +=============== +Giza Background +=============== + +Overview +-------- + +Giza is a collection of tools built around `Sphinx +`_, that coordinates assembling, building, and +deploying documentation. Giza primarily addresses the MongoDB +documentation project; however, its design is sufficiently generic to +be able to facilitate the builds of multiple documentation resources +produced at MongoDB. + +Features and Goals +------------------ + +Giza has the following objectives and primary features: + +Local Build +~~~~~~~~~~~ + +Facilitate fully-local test builds. Contributors to the +documentation should be able to generate the documentation using the +exact same process used to produce the production version of the +resources. + +Content Generation +~~~~~~~~~~~~~~~~~~ + +Generate content from structured forms into reStructuredText that +Sphinx can publish. In an effort to manage duplicated content, and +facilitate sustainable content reuse, Giza translates structured +content, procedures, including command line interfaces, tables of +contents, and API interfaces. + +Parallelism +~~~~~~~~~~~ + +Run multiple Sphinx builds concurrently. Practically speaking, +building the documentation requires running Sphinx multiple +times. Building the documentation requires multiple invocations of +Sphinx, to produce: + +- multiple versions of the manual (i.e. HTML, ``json``, PDF, ePub, + etc.) + +- transitions of the content in different human languages. + +- different editions of a text to address different editions of a + single resource. (e.g. a student and instructor version of a + training resource.) + +Internally, Sphinx itself is not optimally parallelized, and +it's considerably more efficient to run multiple Sphinx processes in +parallel, particularly for larger resources and as the matrix of +required build artifacts grows. diff --git a/giza/docs/source/index.txt b/giza/docs/source/index.txt index 1bc0de7b1..3552ee4f0 100644 --- a/giza/docs/source/index.txt +++ b/giza/docs/source/index.txt @@ -11,5 +11,6 @@ produced at MongoDB. .. toctree:: + /core/giza /tutorial /api diff --git a/giza/docs/source/tutorial.txt b/giza/docs/source/tutorial.txt index 071c8f987..edc3cf65b 100644 --- a/giza/docs/source/tutorial.txt +++ b/giza/docs/source/tutorial.txt @@ -5,3 +5,4 @@ Tutorials .. toctree:: /tutorial/set-up-giza + /tutorial/use-giza diff --git a/giza/docs/source/tutorial/use-giza.txt b/giza/docs/source/tutorial/use-giza.txt new file mode 100644 index 000000000..eae8b395c --- /dev/null +++ b/giza/docs/source/tutorial/use-giza.txt @@ -0,0 +1,189 @@ +========================= +Giza Operations and Tasks +========================= + +Overview +-------- + +Giza provides a large number of operations to support documentation authoring, +production, and deployment. While a ``makefile`` may moderate many +interactions with ``giza``, this document discusses common giza +operations and invocation forms. + +Production Operations +--------------------- + +Sphinx Generation +~~~~~~~~~~~~~~~~~ + +The following commands will build the ``html`` version of the +resource: :: + + giza sphinx --builder html + giza sphinx -b html + +Replace ``html`` with any Sphinx builder you wish to use. You can +specify a list, space separated, of builders to build multiple formats +at once. For example: :: + + giza sphinx --builder latex dirhtml json html singlehtml epub man + +For the MongoDB Manual this is equivalent to the ``make publish`` +operation. For projects that have multiple editions, you can specify +the edition as a section option, for example: :: + + giza sphinx --builder latex json html dirhtml singlehtml --edition saas hosted + +This is the Giza command to build a Manual with two editions, ``saas`` +and ``hosted``, for 5 Sphinx output formats. + +Deploying +~~~~~~~~~ + +There are two targets that deploy built documentation to the +production environment: ``deploy``, which only uploads the +resources; and ``push``, which builds and then deploys the +resources. + +Each branch and repository defines its behavior of the deployment in a +*push* config file. These define a number of "push targets" that +describe how to upload the artifacts. When you run a deploy operation, you +specify one or more of these push targets and Giza will deploy the +artifacts specified in the configuration. + +``push`` takes arguments that are the combination of the ``sphinx`` +command and the ``deploy`` command. Consider the following commands: :: + + giza deploy --target push + giza deploy --target stage + giza deploy --target push-saas + giza deploy --target stage-saas + giza push --deploy push-saas push-hosted --builder latex json html dirhtml singlehtml --edition saas hosted --language es + giza push --deploy push-saas push-hosted --builder latex json html dirhtml singlehtml --edition saas hosted + +Add the ``--dry-run`` or ``-d`` option to any ``deploy`` command to +avoid actually uploading artifacts during testing. + +Git +~~~ + +Giza provides wrappers for several common ``git`` operations. You can +use Giza to apply the patch from a github pull request or from a +single Github commit: :: + + giza git am -p + giza git am --patch + +Replace ```` with the ID of a pull request against the +repository that repository. You can apply any object from github, by +passing a full github URI as the ````. + +All ``giza git`` commands support a ``--branch`` argument that allows +them to perform their operation on a different branch. For example: :: + + giza git am --patch 1066 --branch v4.2 + giza git am -p 1066 -b v4.2 + +You can also cherry-pick commits from the local repository onto the +current branch: :: + giza git cp --commits a5b8087 + giza git cp -c a5b8087 + +The ``git cp`` command allows you to cherry pick a list of commits, +but is most useful in combination with the ``--branch`` option to +apply commits to other branches, as in the following examples: :: + + giza git cp --commits a5b8087 8f9150a 2eb441b + giza git cp -c a5b8087 8f9150a 2eb441b + + giza git cp --commits a5b8087 8f9150a 2eb441b --branch v0.2 + giza git cp -c a5b8087 8f9150a 2eb441b --branch v0.2 + +Development Support Operations +------------------------------ + +Configuration +~~~~~~~~~~~~~ + +You can use the ``giza config`` command to see a rendered version of +the configuration object used during builds. ``config`` allows you to +see how specifying a language or edition will affect the config +object. For example: :: + + giza config + giza config --edition saas + giza config --edition hosted + giza config --edition saas --language es + giza config --edition hosted --language fr + giza config -e saas -l es + giza config -e hosted -l fr + +``generate`` +~~~~~~~~~~~~ + +These operations generate content or fetch data used by the build +without generating the full artifacts. Useful for debugging and +testing. In normal operations the ``sphinx`` operations generate +require inputs, and these operations are not needed. + +``generate`` provides the following operations to generate content +directly: + +- ``api`` +- ``assets`` +- ``images`` +- ``intersphinx`` +- ``redirects`` +- ``options`` +- ``primer`` +- ``steps`` +- ``tables`` +- ``toc`` + +``includes`` +~~~~~~~~~~~~ + +The ``includes`` operations introspect the resources' content reuse, +and allow writers to be able to see the dependency relationship +between source files. ``giza includes`` has the following additional +operations: + +- ``recursive``: returns a list of all files that also include other + files. + +- ``changes``: returns a list of all files in the repository affected + indirectly by uncommitted changes to the current repository. (Requires + `PyGit2 `_) + +- ``once``: returns a list of all included files that are only used + once. + +- ``unused``: returns a list of all included files that are not used + at all. + +- ``graph``: return a document that maps include files to the + dependent source files. Includes the ``--filter`` option, which + allows you to specify a prefix of included files to limit the size + or scope of the graph. + +``package`` +~~~~~~~~~~~ + +Giza provides support for creating "packages" of build artifacts that +you can use to deploy a version of the resource produced on a +different system or at a different time. This makes it possible to +safely deploy a sequence of builds in quick succession. The +``package`` command provides the following options: + +- ``create``: Given a *push target*, build ha package of the current + build output and the current configuration object. + +- ``fetch``: Given a URL, download the package to the + local "build archive." Will refuse to download a package that + already exists in the build archive. + +- ``unwind``: Given a path or URL of a package, extract the package to + the "public output" directory used for staging. + +- ``deploy``: Given a *push target* and the path or URL of a package, + extract the package and upload those artifacts. diff --git a/giza/giza/config/paths.py b/giza/giza/config/paths.py index 20f7ad416..d5dc162a2 100644 --- a/giza/giza/config/paths.py +++ b/giza/giza/config/paths.py @@ -137,8 +137,9 @@ def public_site_output(self): if self.conf.project.branched is True: p.append(self.conf.git.branches.current) - elif (self.conf.project.edition is not None and - self.conf.git.branches.current != self.conf.git.branches.published[0]): + elif (self.conf.git.branches.current != 'master' and + self.conf.projectself.conf.project.edition is not None and + self.conf.git.branches.current != self.conf.git.branches.published[0]): p[-1] += '-' + self.conf.git.branches.current p = os.path.sep.join(p) diff --git a/giza/giza/config/sphinx_config.py b/giza/giza/config/sphinx_config.py index e6ca95978..0e56a92e0 100644 --- a/giza/giza/config/sphinx_config.py +++ b/giza/giza/config/sphinx_config.py @@ -111,7 +111,7 @@ def register(self, builder, language, edition): @property def name(self): if 'name' in self.state: - return self.state['builder'] + return self.state['name'] else: return self.builder diff --git a/giza/giza/content/post/archives.py b/giza/giza/content/post/archives.py index ac3aebb01..1be75c7a1 100644 --- a/giza/giza/content/post/archives.py +++ b/giza/giza/content/post/archives.py @@ -14,6 +14,7 @@ import os.path +from giza.tools.strings import hyph_concat from giza.tools.files import copy_if_needed, create_link, tarball def html_tarball(builder, conf): @@ -25,7 +26,7 @@ def html_tarball(builder, conf): basename = os.path.join(conf.paths.projectroot, conf.paths.public_site_output, - conf.project.name + '-' + conf.git.branches.current) + hyph_concat(conf.project.name, ) + '-' + conf.git.branches.current) tarball_name = basename + '.tar.gz' @@ -46,6 +47,37 @@ def html_tarball(builder, conf): create_link(input_fn=os.path.basename(tarball_name), output_fn=link_name) +def slides_tarball(builder, conf): + copy_if_needed(os.path.join(conf.paths.projectroot, + conf.paths.includes, 'hash.rst'), + os.path.join(conf.paths.projectroot, + conf.paths.branch_output, + builder, 'release.txt')) + + basename = os.path.join(conf.paths.projectroot, + conf.paths.public_site_output, + hyph_concat(conf.project.name, + conf.git.branches.current, + builder)) + + tarball_fn = basename + '.tar.gz' + + tarball(name=tarball_fn, + path=builder, + cdir=os.path.join(conf.paths.projectroot, + conf.paths.branch_output), + newp=os.path.basename(basename)) + + link_name = os.path.join(conf.paths.projectroot, + conf.paths.public_site_output, + hyph_concat(conf.project.name, 'slides') + '.tar.gz') + + if os.path.exists(link_name): + os.remove(link_name) + + create_link(input_fn=os.path.basename(tarball_fn), + output_fn=link_name) + def man_tarball(builder, conf): basename = os.path.join(conf.paths.projectroot, conf.paths.public_site_output, diff --git a/giza/giza/content/sphinx.py b/giza/giza/content/sphinx.py index 2d55ceb61..f575f4cac 100644 --- a/giza/giza/content/sphinx.py +++ b/giza/giza/content/sphinx.py @@ -25,7 +25,7 @@ from giza.tools.strings import timestamp from giza.content.links import create_manual_symlink from giza.content.manpages import manpage_url_tasks -from giza.content.post.archives import man_tarball, html_tarball +from giza.content.post.archives import man_tarball, html_tarball, slides_tarball from giza.content.post.json_output import json_output_tasks from giza.content.post.singlehtml import finalize_single_html_tasks from giza.content.post.gettext import gettext_tasks @@ -208,7 +208,6 @@ def sphinx_tasks(sconf, conf, app): def finalize_sphinx_build(sconf, conf, app): target = sconf.builder - logger.info('starting to finalize the Sphinx build {0}'.format(target)) if target == 'linkcheck': @@ -240,8 +239,13 @@ def finalize_sphinx_build(sconf, conf, app): elif target == 'html': task = app.add('task') task.job = html_tarball - task.args = [target, conf] + task.args = [sconf.name, conf] task.description = "creating tarball for html archive" + elif target == 'slides': + task = app.add('task') + task.job = slides_tarball + task.args = [sconf.name, conf] + task.description = "creating tarball for slides" elif target == 'json': json_output_tasks(conf, app) elif target == 'singlehtml':