Merge pull request #4 from delenamalan/delena-fix-question-bug-2

Fix NavigableString error

setup.py

@@ -61,7 +61,7 @@
     # https://packaging.python.org/en/latest/requirements.html
     install_requires=[
         'mammoth >= 0.3.19, < 0.3.31',
-        'beautifulsoup4 >= 4.4.0',
+        'beautifulsoup4 >= 4.8.2',
     ],
 
     # List additional groups of dependencies here (e.g. development

tests/test_questions.py

@@ -45,6 +45,12 @@ def test_extract_answer_from_html(self):
 
         assert_equal('<ol><li>Total discontinued ammunition still in circulation in the SA Police Service:</li></ol><table><tr><td><p><strong>PROVINCE</strong></p></td><td><p><strong>PREVIOUS QUANTITY</strong></p></td><td><p><strong>CURRENT QUANTITY</strong></p></td></tr><tr><td><p>Western Cape</p></td><td><p>68,542</p></td><td><p>74,627</p></td></tr><tr><td><p>Eastern Cape</p></td><td><p>538,937</p></td><td><p>594,806</p></td></tr><tr><td><p>Northern Cape</p></td><td><p>104,459</p></td><td><p>38,326</p></td></tr><tr><td><p>Free State</p></td><td><p>410,981</p></td><td><p>135,065</p></td></tr><tr><td><p>KwaZulu-Natal</p></td><td><p>523,978</p></td><td><p>228,164</p></td></tr><tr><td><p>North West</p></td><td><p>233,850</p></td><td><p>24,120</p></td></tr><tr><td><p>Mpumalanga</p></td><td><p>296,479</p></td><td><p>92,676</p></td></tr><tr><td><p>Limpopo</p></td><td><p>35,080</p></td><td><p>10,446</p></td></tr><tr><td><p>Gauteng</p></td><td><p>71,764</p></td><td><p>63,209</p></td></tr><tr><td><p>Head Office Divisions</p></td><td><p>2,061,456</p></td><td><p>1,704,286</p></td></tr><tr><td><p><strong>TOTAL:  </strong></p></td><td><p><strong>4,345,526</strong></p></td><td><p><strong>2,965,725</strong></p></td></tr></table><p>This ammunition may only be used for training purposes. However, this ammunition is still in use by Specialized Units within the SAPS for training, ballistic testing of firearms and IBIS test firing purposes. </p><ol><li>An instruction to withdraw all such ammunition was issued from the Divisional Commissioner, Supply Chain Management on 12 September 2006, which also instructed that such ammunition may not be issued any longer as from this date.</li></ol><p>Recently another instruction in regard was issued by the Divisional Commissioner, Supply Chain Management to all Provisional Commissioners and Divisional Commissioners to immediately withdraw all non-standard, obsolete and unserviceable ammunition not in use, from their respective provinces and divisions.</p><p>All non-standard, obsolete and unserviceable ammunition is being sent on a continuous basis to the Ammunition Store at the Division Supply Chain Management for disposal and this is being monitored to ensure compliance.</p>', out)
 
+    def test_extract_answer_from_html_with_different_format(self):
+        html = '<p><strong>NATIONAL ASSEMBLY</strong></p><p><strong>QUESTION </strong><strong>FOR WRITTEN </strong><strong>REPLY</strong></p><h1>QUESTION NO: 1153</h1><p><strong>DATE OF PUBLICATION: </strong><strong> </strong><strong> </strong><strong> 12 JUNE</strong><strong> </strong><strong>2020</strong></p><h2>INTERNAL QUESTION PAPER:  20 OF 2020</h2><p><strong> </strong><strong>Ms T Breedt (FF Plus) </strong><strong>to ask the Minister in </strong><strong>The</strong><strong> Presidency for Women, Youth and Persons with Disabilities</strong><strong>:</strong> </p><p>(1)\tWhether, with reference to her reply to question 937 on 4 June 2020, her Office purchased any goods and/or services below the amount of R500\xa0000 connected to the Covid-19 pandemic; if not, what is the position in this regard; if so, what (a) is the name of each company from which the specified goods and/or services were purchased, (b) is the amount of each transaction and (c) was the service and/or product that each company rendered;</p><p>(2)\twhether there was any deviation from the standard supply chain management procedures in the specified transactions; if so, (a) why and (b) what are the relevant details in each case;</p><p>(3)\twhat were the reasons that the goods and/or services were purchased from the specified companies;</p><p>(4)\twhether she will make a statement on the matter?\tNW1453E</p><p><strong>REPLY</strong></p><table><tr><td><ol><li><strong>Name of Company</strong></li></ol></td><td><ol><li><strong>Amount</strong><strong> transaction</strong></li></ol></td><td><ol><li><strong>Service/Product Rendered</strong></li></ol></td></tr><tr><td><p>Rearabilwe Devoted Creations</p></td><td><p>R25\xa0000.00</p></td><td><p>10 x 25L surface sanitizer</p></td></tr><tr><td><p>Ndamu General Supplier CC</p></td><td><p>R21\xa0000.00</p></td><td><p>Sanitization of Building</p></td></tr><tr><td><p>Manapyane Holdings (Pty) Ltd</p></td><td><p>R11\xa0500.00</p></td><td><p>Infrared Thermometers</p></td></tr></table><ol><li>There were no deviations from the standard supply chain management procedures in the specified transactions.</li></ol><p>(3)\tThe Department utilised the list of service providers as provided by National Treasury.</p><ol><li>No.</li></ol>'
+        out = QuestionAnswerScraper().extract_answer_from_html(html)
+
+        assert_equal('<table><tr><td><ol><li><strong>Name of Company</strong></li></ol></td><td><ol><li><strong>Amount</strong><strong> transaction</strong></li></ol></td><td><ol><li><strong>Service/Product Rendered</strong></li></ol></td></tr><tr><td><p>Rearabilwe Devoted Creations</p></td><td><p>R25\xa0000.00</p></td><td><p>10 x 25L surface sanitizer</p></td></tr><tr><td><p>Ndamu General Supplier CC</p></td><td><p>R21\xa0000.00</p></td><td><p>Sanitization of Building</p></td></tr><tr><td><p>Manapyane Holdings (Pty) Ltd</p></td><td><p>R11\xa0500.00</p></td><td><p>Infrared Thermometers</p></td></tr></table><ol><li>There were no deviations from the standard supply chain management procedures in the specified transactions.</li></ol><p>(3)\tThe Department utilised the list of service providers as provided by National Treasury.</p><ol><li>No.</li></ol>', out)
+
     def test_extract_answer_from_text(self):
         out = self.scraper.extract_answer_from_html('\n36/1/4/1/201500013\nNATIONAL ASSEMBLY\nFOR WRITTEN REPLY\nQUESTION 161\nDATE OF PUBLICATION IN INTERNAL QUESTION PAPER: 12 FEBRUARY 2015 \n(INTERNAL QUESTION PAPER NO 1-2015)\n161. Ms D Kohler (DA) to ask the Minister of Police:\n(a) How many SA Police Service officers in each province currently do not have firearm (i) licenses or (ii) competency certificates and (b) of these, how many still carry firearms?\nNW168E\nREPLY:\n(a) Members of the SAPS, for the purposes of reporting, are categorized under Operational, Support and Management where operational members were first prioritized to obtain their firearm competency. The statistics on 9 February 2015 as per the Training Administration System (TAS) of the SAPS per Province who have not completed the prescribed training are as follows:\n\n\n\n\nPROVINCE\n\n\nTOTAL OPERATIONAL MEMBERS PER PROVINCE\n\n\nNOT COMPLETED\n\n\n% NOT COMPLETED\n\n\n\n\xa0\n\xa0\n\xa0\n\xa0\n\n\n\nWESTERN CAPE\n\n\n15 096\n\n\n872\n\n\n5.78%\n\n\n\n\nEASTERN CAPE\n\n\n15 186\n\n\n988\n\n\n6.51%\n\n\n\n\nNORTHERN CAPE\n\n\n5 215\n\n\n481\n\n\n9.22%\n\n\n\n\nFREE STATE\n\n\n9 404\n\n\n968\n\n\n10.29%\n\n\n\n\nKWAZULU-NATAL\n\n\n18 646\n\n\n406\n\n\n2.18%\n\n\n\n\nNORTH WEST\n\n\n7 522\n\n\n1 040\n\n\n13.83%\n\n\n\n\nMPUMALANGA\n\n\n7 510\n\n\n1 250\n\n\n16.64%\n\n\n\n\nLIMPOPO\n\n\n9 643\n\n\n971\n\n\n10.07%\n\n\n\n\nGAUTENG\n\n\n25 764\n\n\n2 352\n\n\n9.13%\n\n\n\n\nTOTAL\n\n\n113 986\n\n\n9 328\n\n\n8.18%\n\n\n\n\nFrom the 9 328 members, 2 609 are competent in the use of a handgun, 3 182 in the use of a rifle, 4 197 in the use of a shotgun and 4 191 in legal principles.\n(b) 5 728 Members of the 9 328 members who have not completed the prescribed training, according to the TAS system, have been issued with a firearm on their personal inventory.\n')
         assert_equal('\n(a) Members of the SAPS, for the purposes of reporting, are categorized under Operational, Support and Management where operational members were first prioritized to obtain their firearm competency. The statistics on 9 February 2015 as per the Training Administration System (TAS) of the SAPS per Province who have not completed the prescribed training are as follows:\n\n\n\n\nPROVINCE\n\n\nTOTAL OPERATIONAL MEMBERS PER PROVINCE\n\n\nNOT COMPLETED\n\n\n% NOT COMPLETED\n\n\n\n\xa0\n\xa0\n\xa0\n\xa0\n\n\n\nWESTERN CAPE\n\n\n15 096\n\n\n872\n\n\n5.78%\n\n\n\n\nEASTERN CAPE\n\n\n15 186\n\n\n988\n\n\n6.51%\n\n\n\n\nNORTHERN CAPE\n\n\n5 215\n\n\n481\n\n\n9.22%\n\n\n\n\nFREE STATE\n\n\n9 404\n\n\n968\n\n\n10.29%\n\n\n\n\nKWAZULU-NATAL\n\n\n18 646\n\n\n406\n\n\n2.18%\n\n\n\n\nNORTH WEST\n\n\n7 522\n\n\n1 040\n\n\n13.83%\n\n\n\n\nMPUMALANGA\n\n\n7 510\n\n\n1 250\n\n\n16.64%\n\n\n\n\nLIMPOPO\n\n\n9 643\n\n\n971\n\n\n10.07%\n\n\n\n\nGAUTENG\n\n\n25 764\n\n\n2 352\n\n\n9.13%\n\n\n\n\nTOTAL\n\n\n113 986\n\n\n9 328\n\n\n8.18%\n\n\n\n\nFrom the 9 328 members, 2 609 are competent in the use of a handgun, 3 182 in the use of a rifle, 4 197 in the use of a shotgun and 4 191 in legal principles.\n(b) 5 728 Members of the 9 328 members who have not completed the prescribed training, according to the TAS system, have been issued with a firearm on their personal inventory.\n', out)