Deploying to gh-pages from main @ 91e18a9 🚀

Author: rugeli

_modules/cellpack/autopack.html

@@ -86,8 +86,6 @@ <h1>Source code for cellpack.autopack</h1><div class="highlight"><pre>
 <span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">OrderedDict</span>
 <span class="kn">from</span> <span class="nn">pathlib</span> <span class="kn">import</span> <span class="n">Path</span>
 
-<span class="kn">import</span> <span class="nn">boto3</span>
-<span class="kn">import</span> <span class="nn">botocore</span>
 
 <span class="kn">from</span> <span class="nn">cellpack.autopack.DBRecipeHandler</span> <span class="kn">import</span> <span class="n">DBRecipeLoader</span>
 <span class="kn">from</span> <span class="nn">cellpack.autopack.interface_objects.database_ids</span> <span class="kn">import</span> <span class="n">DATABASE_IDS</span>
@@ -310,23 +308,6 @@ <h1>Source code for cellpack.autopack</h1><div class="highlight"><pre>
 
 
 
-<div class="viewcode-block" id="download_file_from_s3">
-<a class="viewcode-back" href="../../cellpack.autopack.html#cellpack.autopack.download_file_from_s3">[docs]</a>
-<span class="k">def</span> <span class="nf">download_file_from_s3</span><span class="p">(</span><span class="n">s3_uri</span><span class="p">,</span> <span class="n">local_file_path</span><span class="p">):</span>
-    <span class="n">s3_client</span> <span class="o">=</span> <span class="n">boto3</span><span class="o">.</span><span class="n">client</span><span class="p">(</span><span class="s2">&quot;s3&quot;</span><span class="p">)</span>
-    <span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span> <span class="o">=</span> <span class="n">parse_s3_uri</span><span class="p">(</span><span class="n">s3_uri</span><span class="p">)</span>
-
-    <span class="k">try</span><span class="p">:</span>
-        <span class="n">s3_client</span><span class="o">.</span><span class="n">download_file</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">local_file_path</span><span class="p">)</span>
-        <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;File downloaded successfully.&quot;</span><span class="p">)</span>
-    <span class="k">except</span> <span class="n">botocore</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">ClientError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
-        <span class="k">if</span> <span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="p">[</span><span class="s2">&quot;Error&quot;</span><span class="p">][</span><span class="s2">&quot;Code&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="s2">&quot;404&quot;</span><span class="p">:</span>
-            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;The object does not exist.&quot;</span><span class="p">)</span>
-        <span class="k">else</span><span class="p">:</span>
-            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;An error occurred while downloading the file.&quot;</span><span class="p">)</span></div>
-
-
-
 <div class="viewcode-block" id="parse_s3_uri">
 <a class="viewcode-back" href="../../cellpack.autopack.html#cellpack.autopack.parse_s3_uri">[docs]</a>
 <span class="k">def</span> <span class="nf">parse_s3_uri</span><span class="p">(</span><span class="n">s3_uri</span><span class="p">):</span>
@@ -341,25 +322,23 @@ <h1>Source code for cellpack.autopack</h1><div class="highlight"><pre>
 
 
 
+<div class="viewcode-block" id="is_s3_url">
+<a class="viewcode-back" href="../../cellpack.autopack.html#cellpack.autopack.is_s3_url">[docs]</a>
+<span class="k">def</span> <span class="nf">is_s3_url</span><span class="p">(</span><span class="n">file_path</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">file_path</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s2">&quot;s3://&quot;</span><span class="p">)</span> <span class="o">!=</span> <span class="o">-</span><span class="mi">1</span></div>
+
+
+
 <div class="viewcode-block" id="download_file">
 <a class="viewcode-back" href="../../cellpack.autopack.html#cellpack.autopack.download_file">[docs]</a>
-<span class="k">def</span> <span class="nf">download_file</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">local_file_path</span><span class="p">,</span> <span class="n">reporthook</span><span class="p">):</span>
+<span class="k">def</span> <span class="nf">download_file</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">local_file_path</span><span class="p">,</span> <span class="n">reporthook</span><span class="p">,</span> <span class="n">database_name</span><span class="o">=</span><span class="s2">&quot;aws&quot;</span><span class="p">):</span>
     <span class="k">if</span> <span class="n">is_s3_url</span><span class="p">(</span><span class="n">url</span><span class="p">):</span>
-        <span class="c1"># download from s3</span>
-        <span class="c1"># bucket_name, folder, key = parse_s3_uri(url)</span>
-        <span class="c1"># s3_handler = DATABASE_IDS.handlers().get(DATABASE_IDS.AWS)</span>
-        <span class="c1"># s3_handler = s3_handler(bucket_name, folder)</span>
-        <span class="n">s3_client</span> <span class="o">=</span> <span class="n">boto3</span><span class="o">.</span><span class="n">client</span><span class="p">(</span><span class="s2">&quot;s3&quot;</span><span class="p">)</span>
+        <span class="n">db</span> <span class="o">=</span> <span class="n">DATABASE_IDS</span><span class="o">.</span><span class="n">handlers</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">database_name</span><span class="p">)</span>
         <span class="n">bucket_name</span><span class="p">,</span> <span class="n">folder</span><span class="p">,</span> <span class="n">key</span> <span class="o">=</span> <span class="n">parse_s3_uri</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
-        <span class="k">try</span><span class="p">:</span>
-            <span class="n">s3_client</span><span class="o">.</span><span class="n">download_file</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">folder</span><span class="si">}</span><span class="s2">/</span><span class="si">{</span><span class="n">key</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">local_file_path</span><span class="p">)</span>
-            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;File downloaded successfully.&quot;</span><span class="p">)</span>
-        <span class="k">except</span> <span class="n">botocore</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">ClientError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
-            <span class="k">if</span> <span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="p">[</span><span class="s2">&quot;Error&quot;</span><span class="p">][</span><span class="s2">&quot;Code&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="s2">&quot;404&quot;</span><span class="p">:</span>
-                <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;The object does not exist.&quot;</span><span class="p">)</span>
-            <span class="k">else</span><span class="p">:</span>
-                <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;An error occurred while downloading the file.&quot;</span><span class="p">)</span>
-
+        <span class="n">initialize_db</span> <span class="o">=</span> <span class="n">db</span><span class="p">(</span>
+            <span class="n">bucket_name</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">sub_folder_name</span><span class="o">=</span><span class="n">folder</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="s2">&quot;us-west-2&quot;</span>
+        <span class="p">)</span>
+        <span class="n">initialize_db</span><span class="o">.</span><span class="n">download_file</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">folder</span><span class="si">}</span><span class="s2">/</span><span class="si">{</span><span class="n">key</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">local_file_path</span><span class="p">)</span>
     <span class="k">elif</span> <span class="n">url_exists</span><span class="p">(</span><span class="n">url</span><span class="p">):</span>
         <span class="k">try</span><span class="p">:</span>
             <span class="n">urllib</span><span class="o">.</span><span class="n">urlretrieve</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">local_file_path</span><span class="p">,</span> <span class="n">reporthook</span><span class="o">=</span><span class="n">reporthook</span><span class="p">)</span>
@@ -380,13 +359,6 @@ <h1>Source code for cellpack.autopack</h1><div class="highlight"><pre>
 
 
 
-<div class="viewcode-block" id="is_s3_url">
-<a class="viewcode-back" href="../../cellpack.autopack.html#cellpack.autopack.is_s3_url">[docs]</a>
-<span class="k">def</span> <span class="nf">is_s3_url</span><span class="p">(</span><span class="n">file_path</span><span class="p">):</span>
-    <span class="k">return</span> <span class="n">file_path</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s2">&quot;s3://&quot;</span><span class="p">)</span> <span class="o">!=</span> <span class="o">-</span><span class="mi">1</span></div>
-
-
-
 <div class="viewcode-block" id="is_remote_path">
 <a class="viewcode-back" href="../../cellpack.autopack.html#cellpack.autopack.is_remote_path">[docs]</a>
 <span class="k">def</span> <span class="nf">is_remote_path</span><span class="p">(</span><span class="n">file_path</span><span class="p">):</span>

_modules/cellpack/autopack/AWSHandler.html

@@ -61,7 +61,7 @@ <h1>Source code for cellpack.autopack.AWSHandler</h1><div class="highlight"><pre
         <span class="bp">self</span><span class="p">,</span>
         <span class="n">bucket_name</span><span class="p">,</span>
         <span class="n">sub_folder_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-        <span class="n">region_name</span><span class="o">=</span><span class="s2">&quot;us-west-2&quot;</span><span class="p">,</span>
+        <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="p">):</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">bucket_name</span> <span class="o">=</span> <span class="n">bucket_name</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">folder_name</span> <span class="o">=</span> <span class="n">sub_folder_name</span>
@@ -117,6 +117,25 @@ <h1>Source code for cellpack.autopack.AWSHandler</h1><div class="highlight"><pre
         <span class="k">return</span> <span class="n">file_name</span></div>
 
 
+<div class="viewcode-block" id="AWSHandler.download_file">
+<a class="viewcode-back" href="../../../cellpack.autopack.html#cellpack.autopack.AWSHandler.AWSHandler.download_file">[docs]</a>
+    <span class="k">def</span> <span class="nf">download_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">local_file_path</span><span class="p">):</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Download a file from S3</span>
+<span class="sd">        :param key: S3 object key</span>
+<span class="sd">        :param local_file_path: Local file path to save the downloaded file</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="k">try</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">s3_client</span><span class="o">.</span><span class="n">download_file</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">local_file_path</span><span class="p">)</span>
+            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;File downloaded successfully.&quot;</span><span class="p">)</span>
+        <span class="k">except</span> <span class="n">ClientError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="p">[</span><span class="s2">&quot;Error&quot;</span><span class="p">][</span><span class="s2">&quot;Code&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="s2">&quot;404&quot;</span><span class="p">:</span>
+                <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;The object does not exist.&quot;</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;An error occurred while downloading the file.&quot;</span><span class="p">)</span></div>
+
+
 <div class="viewcode-block" id="AWSHandler.create_presigned_url">
 <a class="viewcode-back" href="../../../cellpack.autopack.html#cellpack.autopack.AWSHandler.AWSHandler.create_presigned_url">[docs]</a>
     <span class="k">def</span> <span class="nf">create_presigned_url</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">object_name</span><span class="p">,</span> <span class="n">expiration</span><span class="o">=</span><span class="mi">3600</span><span class="p">):</span>