From 08e32c90cf6f57722bda4d315642b218c3fa8d4f Mon Sep 17 00:00:00 2001 From: Delena Malan Date: Tue, 4 Feb 2020 16:45:09 +0200 Subject: [PATCH] Fix unicode issues for Python3 and drop Python 2 --- .travis.yml | 1 - tests/test_questions.py | 12 ++++----- za_parliament_scrapers/questions.py | 42 ++++++++++++++--------------- 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2add790..4550208 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: python python: - - "2.7" - "3.7" install: diff --git a/tests/test_questions.py b/tests/test_questions.py index a38ab2d..6b2ae71 100644 --- a/tests/test_questions.py +++ b/tests/test_questions.py @@ -24,16 +24,16 @@ def test_simple(self): def test_extract_from_document(self): text, html = self.scraper.extract_content_from_document("tests/fixtures/RNW126-150302.docx") - assert_equal(u'36/1/4/1/201500010\n\nNATIONAL ASSEMBLY\n\n\n\nFOR WRITTEN REPLY\n\n\n\nQUESTION 126\n\n\n\nDATE OF PUBLICATION IN INTERNAL QUESTION PAPER: 12 FEBRUARY 2015 \n\n(INTERNAL QUESTION PAPER NO 1-2015)\n\n\n\n126. Ms D Kohler (DA) to ask the Minister of Police:\n\nWith reference to the reply to question 228 on 18 March 2014, (a) how much discontinued ammunition is still in circulation in the SA Police Service in each province and (b) what action is to be taken to remove this ammunition from use?\n\nNW131E\n\nREPLY:\n\nTotal discontinued ammunition still in circulation in the SA Police Service:\n\n\n\nPROVINCE\n\nPREVIOUS QUANTITY\n\nCURRENT QUANTITY\n\nWestern Cape\n\n68,542\n\n74,627\n\nEastern Cape\n\n538,937\n\n594,806\n\nNorthern Cape\n\n104,459\n\n38,326\n\nFree State\n\n410,981\n\n135,065\n\nKwaZulu-Natal\n\n523,978\n\n228,164\n\nNorth West\n\n233,850\n\n24,120\n\nMpumalanga\n\n296,479\n\n92,676\n\nLimpopo\n\n35,080\n\n10,446\n\nGauteng\n\n71,764\n\n63,209\n\nHead Office Divisions\n\n2,061,456\n\n1,704,286\n\nTOTAL: \n\n4,345,526\n\n2,965,725\n\nThis ammunition may only be used for training purposes. However, this ammunition is still in use by Specialized Units within the SAPS for training, ballistic testing of firearms and IBIS test firing purposes. \n\nAn instruction to withdraw all such ammunition was issued from the Divisional Commissioner, Supply Chain Management on 12 September 2006, which also instructed that such ammunition may not be issued any longer as from this date.\n\nRecently another instruction in regard was issued by the Divisional Commissioner, Supply Chain Management to all Provisional Commissioners and Divisional Commissioners to immediately withdraw all non-standard, obsolete and unserviceable ammunition not in use, from their respective provinces and divisions.\n\nAll non-standard, obsolete and unserviceable ammunition is being sent on a continuous basis to the Ammunition Store at the Division Supply Chain Management for disposal and this is being monitored to ensure compliance.\n\n\n\n\n\n\n\n\n\n', text) + assert_equal('36/1/4/1/201500010\n\nNATIONAL ASSEMBLY\n\n\n\nFOR WRITTEN REPLY\n\n\n\nQUESTION 126\n\n\n\nDATE OF PUBLICATION IN INTERNAL QUESTION PAPER: 12 FEBRUARY 2015 \n\n(INTERNAL QUESTION PAPER NO 1-2015)\n\n\n\n126. Ms D Kohler (DA) to ask the Minister of Police:\n\nWith reference to the reply to question 228 on 18 March 2014, (a) how much discontinued ammunition is still in circulation in the SA Police Service in each province and (b) what action is to be taken to remove this ammunition from use?\n\nNW131E\n\nREPLY:\n\nTotal discontinued ammunition still in circulation in the SA Police Service:\n\n\n\nPROVINCE\n\nPREVIOUS QUANTITY\n\nCURRENT QUANTITY\n\nWestern Cape\n\n68,542\n\n74,627\n\nEastern Cape\n\n538,937\n\n594,806\n\nNorthern Cape\n\n104,459\n\n38,326\n\nFree State\n\n410,981\n\n135,065\n\nKwaZulu-Natal\n\n523,978\n\n228,164\n\nNorth West\n\n233,850\n\n24,120\n\nMpumalanga\n\n296,479\n\n92,676\n\nLimpopo\n\n35,080\n\n10,446\n\nGauteng\n\n71,764\n\n63,209\n\nHead Office Divisions\n\n2,061,456\n\n1,704,286\n\nTOTAL: \n\n4,345,526\n\n2,965,725\n\nThis ammunition may only be used for training purposes. However, this ammunition is still in use by Specialized Units within the SAPS for training, ballistic testing of firearms and IBIS test firing purposes. \n\nAn instruction to withdraw all such ammunition was issued from the Divisional Commissioner, Supply Chain Management on 12 September 2006, which also instructed that such ammunition may not be issued any longer as from this date.\n\nRecently another instruction in regard was issued by the Divisional Commissioner, Supply Chain Management to all Provisional Commissioners and Divisional Commissioners to immediately withdraw all non-standard, obsolete and unserviceable ammunition not in use, from their respective provinces and divisions.\n\nAll non-standard, obsolete and unserviceable ammunition is being sent on a continuous basis to the Ammunition Store at the Division Supply Chain Management for disposal and this is being monitored to ensure compliance.\n\n\n\n\n\n\n\n\n\n', text) - assert_equal(u'

36/1/4/1/201500010

NATIONAL ASSEMBLY

FOR WRITTEN REPLY

QUESTION 126

DATE OF PUBLICATION IN INTERNAL QUESTION PAPER: 12 FEBRUARY 2015

(INTERNAL QUESTION PAPER NO 1-2015)

126. Ms D Kohler (DA) to ask the Minister of Police:

With reference to the reply to question 228 on 18 March 2014, (a) how much discontinued ammunition is still in circulation in the SA Police Service in each province and (b) what action is to be taken to remove this ammunition from use?

NW131E

REPLY:

  1. Total discontinued ammunition still in circulation in the SA Police Service:

PROVINCE

PREVIOUS QUANTITY

CURRENT QUANTITY

Western Cape

68,542

74,627

Eastern Cape

538,937

594,806

Northern Cape

104,459

38,326

Free State

410,981

135,065

KwaZulu-Natal

523,978

228,164

North West

233,850

24,120

Mpumalanga

296,479

92,676

Limpopo

35,080

10,446

Gauteng

71,764

63,209

Head Office Divisions

2,061,456

1,704,286

TOTAL:

4,345,526

2,965,725

This ammunition may only be used for training purposes. However, this ammunition is still in use by Specialized Units within the SAPS for training, ballistic testing of firearms and IBIS test firing purposes.

  1. An instruction to withdraw all such ammunition was issued from the Divisional Commissioner, Supply Chain Management on 12 September 2006, which also instructed that such ammunition may not be issued any longer as from this date.

Recently another instruction in regard was issued by the Divisional Commissioner, Supply Chain Management to all Provisional Commissioners and Divisional Commissioners to immediately withdraw all non-standard, obsolete and unserviceable ammunition not in use, from their respective provinces and divisions.

All non-standard, obsolete and unserviceable ammunition is being sent on a continuous basis to the Ammunition Store at the Division Supply Chain Management for disposal and this is being monitored to ensure compliance.

', html) + assert_equal('

36/1/4/1/201500010

NATIONAL ASSEMBLY

FOR WRITTEN REPLY

QUESTION 126

DATE OF PUBLICATION IN INTERNAL QUESTION PAPER: 12 FEBRUARY 2015

(INTERNAL QUESTION PAPER NO 1-2015)

126. Ms D Kohler (DA) to ask the Minister of Police:

With reference to the reply to question 228 on 18 March 2014, (a) how much discontinued ammunition is still in circulation in the SA Police Service in each province and (b) what action is to be taken to remove this ammunition from use?

NW131E

REPLY:

  1. Total discontinued ammunition still in circulation in the SA Police Service:

PROVINCE

PREVIOUS QUANTITY

CURRENT QUANTITY

Western Cape

68,542

74,627

Eastern Cape

538,937

594,806

Northern Cape

104,459

38,326

Free State

410,981

135,065

KwaZulu-Natal

523,978

228,164

North West

233,850

24,120

Mpumalanga

296,479

92,676

Limpopo

35,080

10,446

Gauteng

71,764

63,209

Head Office Divisions

2,061,456

1,704,286

TOTAL:

4,345,526

2,965,725

This ammunition may only be used for training purposes. However, this ammunition is still in use by Specialized Units within the SAPS for training, ballistic testing of firearms and IBIS test firing purposes.

  1. An instruction to withdraw all such ammunition was issued from the Divisional Commissioner, Supply Chain Management on 12 September 2006, which also instructed that such ammunition may not be issued any longer as from this date.

Recently another instruction in regard was issued by the Divisional Commissioner, Supply Chain Management to all Provisional Commissioners and Divisional Commissioners to immediately withdraw all non-standard, obsolete and unserviceable ammunition not in use, from their respective provinces and divisions.

All non-standard, obsolete and unserviceable ammunition is being sent on a continuous basis to the Ammunition Store at the Division Supply Chain Management for disposal and this is being monitored to ensure compliance.

', html) def test_extract_answer_from_html(self): - html = u'

36/1/4/1/201500010

NATIONAL ASSEMBLY

FOR WRITTEN REPLY

QUESTION 126

DATE OF PUBLICATION IN INTERNAL QUESTION PAPER: 12 FEBRUARY 2015

(INTERNAL QUESTION PAPER NO 1-2015)

126. Ms D Kohler (DA) to ask the Minister of Police:

With reference to the reply to question 228 on 18 March 2014, (a) how much discontinued ammunition is still in circulation in the SA Police Service in each province and (b) what action is to be taken to remove this ammunition from use?

NW131E

REPLY:

  1. Total discontinued ammunition still in circulation in the SA Police Service:

PROVINCE

PREVIOUS QUANTITY

CURRENT QUANTITY

Western Cape

68,542

74,627

Eastern Cape

538,937

594,806

Northern Cape

104,459

38,326

Free State

410,981

135,065

KwaZulu-Natal

523,978

228,164

North West

233,850

24,120

Mpumalanga

296,479

92,676

Limpopo

35,080

10,446

Gauteng

71,764

63,209

Head Office Divisions

2,061,456

1,704,286

TOTAL:

4,345,526

2,965,725

This ammunition may only be used for training purposes. However, this ammunition is still in use by Specialized Units within the SAPS for training, ballistic testing of firearms and IBIS test firing purposes.

  1. An instruction to withdraw all such ammunition was issued from the Divisional Commissioner, Supply Chain Management on 12 September 2006, which also instructed that such ammunition may not be issued any longer as from this date.

Recently another instruction in regard was issued by the Divisional Commissioner, Supply Chain Management to all Provisional Commissioners and Divisional Commissioners to immediately withdraw all non-standard, obsolete and unserviceable ammunition not in use, from their respective provinces and divisions.

All non-standard, obsolete and unserviceable ammunition is being sent on a continuous basis to the Ammunition Store at the Division Supply Chain Management for disposal and this is being monitored to ensure compliance.

' + html = '

36/1/4/1/201500010

NATIONAL ASSEMBLY

FOR WRITTEN REPLY

QUESTION 126

DATE OF PUBLICATION IN INTERNAL QUESTION PAPER: 12 FEBRUARY 2015

(INTERNAL QUESTION PAPER NO 1-2015)

126. Ms D Kohler (DA) to ask the Minister of Police:

With reference to the reply to question 228 on 18 March 2014, (a) how much discontinued ammunition is still in circulation in the SA Police Service in each province and (b) what action is to be taken to remove this ammunition from use?

NW131E

REPLY:

  1. Total discontinued ammunition still in circulation in the SA Police Service:

PROVINCE

PREVIOUS QUANTITY

CURRENT QUANTITY

Western Cape

68,542

74,627

Eastern Cape

538,937

594,806

Northern Cape

104,459

38,326

Free State

410,981

135,065

KwaZulu-Natal

523,978

228,164

North West

233,850

24,120

Mpumalanga

296,479

92,676

Limpopo

35,080

10,446

Gauteng

71,764

63,209

Head Office Divisions

2,061,456

1,704,286

TOTAL:

4,345,526

2,965,725

This ammunition may only be used for training purposes. However, this ammunition is still in use by Specialized Units within the SAPS for training, ballistic testing of firearms and IBIS test firing purposes.

  1. An instruction to withdraw all such ammunition was issued from the Divisional Commissioner, Supply Chain Management on 12 September 2006, which also instructed that such ammunition may not be issued any longer as from this date.

Recently another instruction in regard was issued by the Divisional Commissioner, Supply Chain Management to all Provisional Commissioners and Divisional Commissioners to immediately withdraw all non-standard, obsolete and unserviceable ammunition not in use, from their respective provinces and divisions.

All non-standard, obsolete and unserviceable ammunition is being sent on a continuous basis to the Ammunition Store at the Division Supply Chain Management for disposal and this is being monitored to ensure compliance.

' out = self.scraper.extract_answer_from_html(html) - assert_equal(u'
  1. Total discontinued ammunition still in circulation in the SA Police Service:

PROVINCE

PREVIOUS QUANTITY

CURRENT QUANTITY

Western Cape

68,542

74,627

Eastern Cape

538,937

594,806

Northern Cape

104,459

38,326

Free State

410,981

135,065

KwaZulu-Natal

523,978

228,164

North West

233,850

24,120

Mpumalanga

296,479

92,676

Limpopo

35,080

10,446

Gauteng

71,764

63,209

Head Office Divisions

2,061,456

1,704,286

TOTAL:

4,345,526

2,965,725

This ammunition may only be used for training purposes. However, this ammunition is still in use by Specialized Units within the SAPS for training, ballistic testing of firearms and IBIS test firing purposes.

  1. An instruction to withdraw all such ammunition was issued from the Divisional Commissioner, Supply Chain Management on 12 September 2006, which also instructed that such ammunition may not be issued any longer as from this date.

Recently another instruction in regard was issued by the Divisional Commissioner, Supply Chain Management to all Provisional Commissioners and Divisional Commissioners to immediately withdraw all non-standard, obsolete and unserviceable ammunition not in use, from their respective provinces and divisions.

All non-standard, obsolete and unserviceable ammunition is being sent on a continuous basis to the Ammunition Store at the Division Supply Chain Management for disposal and this is being monitored to ensure compliance.

', out) + assert_equal('
  1. Total discontinued ammunition still in circulation in the SA Police Service:

PROVINCE

PREVIOUS QUANTITY

CURRENT QUANTITY

Western Cape

68,542

74,627

Eastern Cape

538,937

594,806

Northern Cape

104,459

38,326

Free State

410,981

135,065

KwaZulu-Natal

523,978

228,164

North West

233,850

24,120

Mpumalanga

296,479

92,676

Limpopo

35,080

10,446

Gauteng

71,764

63,209

Head Office Divisions

2,061,456

1,704,286

TOTAL:

4,345,526

2,965,725

This ammunition may only be used for training purposes. However, this ammunition is still in use by Specialized Units within the SAPS for training, ballistic testing of firearms and IBIS test firing purposes.

  1. An instruction to withdraw all such ammunition was issued from the Divisional Commissioner, Supply Chain Management on 12 September 2006, which also instructed that such ammunition may not be issued any longer as from this date.

Recently another instruction in regard was issued by the Divisional Commissioner, Supply Chain Management to all Provisional Commissioners and Divisional Commissioners to immediately withdraw all non-standard, obsolete and unserviceable ammunition not in use, from their respective provinces and divisions.

All non-standard, obsolete and unserviceable ammunition is being sent on a continuous basis to the Ammunition Store at the Division Supply Chain Management for disposal and this is being monitored to ensure compliance.

', out) def test_extract_answer_from_text(self): - out = self.scraper.extract_answer_from_html(u'\n36/1/4/1/201500013\nNATIONAL ASSEMBLY\nFOR WRITTEN REPLY\nQUESTION 161\nDATE OF PUBLICATION IN INTERNAL QUESTION PAPER: 12 FEBRUARY 2015 \n(INTERNAL QUESTION PAPER NO 1-2015)\n161. Ms D Kohler (DA) to ask the Minister of Police:\n(a) How many SA Police Service officers in each province currently do not have firearm (i) licenses or (ii) competency certificates and (b) of these, how many still carry firearms?\nNW168E\nREPLY:\n(a) Members of the SAPS, for the purposes of reporting, are categorized under Operational, Support and Management where operational members were first prioritized to obtain their firearm competency. The statistics on 9 February 2015 as per the Training Administration System (TAS) of the SAPS per Province who have not completed the prescribed training are as follows:\n\n\n\n\nPROVINCE\n\n\nTOTAL OPERATIONAL MEMBERS PER PROVINCE\n\n\nNOT COMPLETED\n\n\n% NOT COMPLETED\n\n\n\n\xa0\n\xa0\n\xa0\n\xa0\n\n\n\nWESTERN CAPE\n\n\n15 096\n\n\n872\n\n\n5.78%\n\n\n\n\nEASTERN CAPE\n\n\n15 186\n\n\n988\n\n\n6.51%\n\n\n\n\nNORTHERN CAPE\n\n\n5 215\n\n\n481\n\n\n9.22%\n\n\n\n\nFREE STATE\n\n\n9 404\n\n\n968\n\n\n10.29%\n\n\n\n\nKWAZULU-NATAL\n\n\n18 646\n\n\n406\n\n\n2.18%\n\n\n\n\nNORTH WEST\n\n\n7 522\n\n\n1 040\n\n\n13.83%\n\n\n\n\nMPUMALANGA\n\n\n7 510\n\n\n1 250\n\n\n16.64%\n\n\n\n\nLIMPOPO\n\n\n9 643\n\n\n971\n\n\n10.07%\n\n\n\n\nGAUTENG\n\n\n25 764\n\n\n2 352\n\n\n9.13%\n\n\n\n\nTOTAL\n\n\n113 986\n\n\n9 328\n\n\n8.18%\n\n\n\n\nFrom the 9 328 members, 2 609 are competent in the use of a handgun, 3 182 in the use of a rifle, 4 197 in the use of a shotgun and 4 191 in legal principles.\n(b) 5 728 Members of the 9 328 members who have not completed the prescribed training, according to the TAS system, have been issued with a firearm on their personal inventory.\n') - assert_equal(u'\n(a) Members of the SAPS, for the purposes of reporting, are categorized under Operational, Support and Management where operational members were first prioritized to obtain their firearm competency. The statistics on 9 February 2015 as per the Training Administration System (TAS) of the SAPS per Province who have not completed the prescribed training are as follows:\n\n\n\n\nPROVINCE\n\n\nTOTAL OPERATIONAL MEMBERS PER PROVINCE\n\n\nNOT COMPLETED\n\n\n% NOT COMPLETED\n\n\n\n\xa0\n\xa0\n\xa0\n\xa0\n\n\n\nWESTERN CAPE\n\n\n15 096\n\n\n872\n\n\n5.78%\n\n\n\n\nEASTERN CAPE\n\n\n15 186\n\n\n988\n\n\n6.51%\n\n\n\n\nNORTHERN CAPE\n\n\n5 215\n\n\n481\n\n\n9.22%\n\n\n\n\nFREE STATE\n\n\n9 404\n\n\n968\n\n\n10.29%\n\n\n\n\nKWAZULU-NATAL\n\n\n18 646\n\n\n406\n\n\n2.18%\n\n\n\n\nNORTH WEST\n\n\n7 522\n\n\n1 040\n\n\n13.83%\n\n\n\n\nMPUMALANGA\n\n\n7 510\n\n\n1 250\n\n\n16.64%\n\n\n\n\nLIMPOPO\n\n\n9 643\n\n\n971\n\n\n10.07%\n\n\n\n\nGAUTENG\n\n\n25 764\n\n\n2 352\n\n\n9.13%\n\n\n\n\nTOTAL\n\n\n113 986\n\n\n9 328\n\n\n8.18%\n\n\n\n\nFrom the 9 328 members, 2 609 are competent in the use of a handgun, 3 182 in the use of a rifle, 4 197 in the use of a shotgun and 4 191 in legal principles.\n(b) 5 728 Members of the 9 328 members who have not completed the prescribed training, according to the TAS system, have been issued with a firearm on their personal inventory.\n', out) + out = self.scraper.extract_answer_from_html('\n36/1/4/1/201500013\nNATIONAL ASSEMBLY\nFOR WRITTEN REPLY\nQUESTION 161\nDATE OF PUBLICATION IN INTERNAL QUESTION PAPER: 12 FEBRUARY 2015 \n(INTERNAL QUESTION PAPER NO 1-2015)\n161. Ms D Kohler (DA) to ask the Minister of Police:\n(a) How many SA Police Service officers in each province currently do not have firearm (i) licenses or (ii) competency certificates and (b) of these, how many still carry firearms?\nNW168E\nREPLY:\n(a) Members of the SAPS, for the purposes of reporting, are categorized under Operational, Support and Management where operational members were first prioritized to obtain their firearm competency. The statistics on 9 February 2015 as per the Training Administration System (TAS) of the SAPS per Province who have not completed the prescribed training are as follows:\n\n\n\n\nPROVINCE\n\n\nTOTAL OPERATIONAL MEMBERS PER PROVINCE\n\n\nNOT COMPLETED\n\n\n% NOT COMPLETED\n\n\n\n\xa0\n\xa0\n\xa0\n\xa0\n\n\n\nWESTERN CAPE\n\n\n15 096\n\n\n872\n\n\n5.78%\n\n\n\n\nEASTERN CAPE\n\n\n15 186\n\n\n988\n\n\n6.51%\n\n\n\n\nNORTHERN CAPE\n\n\n5 215\n\n\n481\n\n\n9.22%\n\n\n\n\nFREE STATE\n\n\n9 404\n\n\n968\n\n\n10.29%\n\n\n\n\nKWAZULU-NATAL\n\n\n18 646\n\n\n406\n\n\n2.18%\n\n\n\n\nNORTH WEST\n\n\n7 522\n\n\n1 040\n\n\n13.83%\n\n\n\n\nMPUMALANGA\n\n\n7 510\n\n\n1 250\n\n\n16.64%\n\n\n\n\nLIMPOPO\n\n\n9 643\n\n\n971\n\n\n10.07%\n\n\n\n\nGAUTENG\n\n\n25 764\n\n\n2 352\n\n\n9.13%\n\n\n\n\nTOTAL\n\n\n113 986\n\n\n9 328\n\n\n8.18%\n\n\n\n\nFrom the 9 328 members, 2 609 are competent in the use of a handgun, 3 182 in the use of a rifle, 4 197 in the use of a shotgun and 4 191 in legal principles.\n(b) 5 728 Members of the 9 328 members who have not completed the prescribed training, according to the TAS system, have been issued with a firearm on their personal inventory.\n') + assert_equal('\n(a) Members of the SAPS, for the purposes of reporting, are categorized under Operational, Support and Management where operational members were first prioritized to obtain their firearm competency. The statistics on 9 February 2015 as per the Training Administration System (TAS) of the SAPS per Province who have not completed the prescribed training are as follows:\n\n\n\n\nPROVINCE\n\n\nTOTAL OPERATIONAL MEMBERS PER PROVINCE\n\n\nNOT COMPLETED\n\n\n% NOT COMPLETED\n\n\n\n\xa0\n\xa0\n\xa0\n\xa0\n\n\n\nWESTERN CAPE\n\n\n15 096\n\n\n872\n\n\n5.78%\n\n\n\n\nEASTERN CAPE\n\n\n15 186\n\n\n988\n\n\n6.51%\n\n\n\n\nNORTHERN CAPE\n\n\n5 215\n\n\n481\n\n\n9.22%\n\n\n\n\nFREE STATE\n\n\n9 404\n\n\n968\n\n\n10.29%\n\n\n\n\nKWAZULU-NATAL\n\n\n18 646\n\n\n406\n\n\n2.18%\n\n\n\n\nNORTH WEST\n\n\n7 522\n\n\n1 040\n\n\n13.83%\n\n\n\n\nMPUMALANGA\n\n\n7 510\n\n\n1 250\n\n\n16.64%\n\n\n\n\nLIMPOPO\n\n\n9 643\n\n\n971\n\n\n10.07%\n\n\n\n\nGAUTENG\n\n\n25 764\n\n\n2 352\n\n\n9.13%\n\n\n\n\nTOTAL\n\n\n113 986\n\n\n9 328\n\n\n8.18%\n\n\n\n\nFrom the 9 328 members, 2 609 are competent in the use of a handgun, 3 182 in the use of a rifle, 4 197 in the use of a shotgun and 4 191 in legal principles.\n(b) 5 728 Members of the 9 328 members who have not completed the prescribed training, according to the TAS system, have been issued with a firearm on their personal inventory.\n', out) diff --git a/za_parliament_scrapers/questions.py b/za_parliament_scrapers/questions.py index 86a6b62..001272e 100644 --- a/za_parliament_scrapers/questions.py +++ b/za_parliament_scrapers/questions.py @@ -15,7 +15,7 @@ def strip_dict(d): >>> d = {'a': ' foo ', 'b': 3, 'c': ' bar'} >>> result = strip_dict(d) >>> type(result) - + >>> sorted(result.items()) [('a', 'foo'), ('b', 3), ('c', 'bar')] """ @@ -30,7 +30,7 @@ class QuestionAnswerScraper(object): BAR_REGEX = re.compile(r'^_+$', re.MULTILINE) QUESTION_RE = re.compile( - ur""" + r""" (?P (?:(?P\d+)\.?\s+)? # Question number (?P[-\w\s]+?) # Name of question asker @@ -125,61 +125,61 @@ def extract_questions_from_text(self, text): # Checks for QUESTION_RE # Shows the need for - in the party - >>> qn = u'144. Mr D B Feldman (COPE-Gauteng) to ask the Minister of Defence and Military Veterans: Whether the deployment of the SA National Defence Force soldiers to the Central African Republic and the Democratic Republic of Congo is in line with our international policy with regard to (a) upholding international peace, (b) the promotion of constitutional democracy and (c) the respect for parliamentary democracy; if not, why not; if so, what are the (i) policies which underpin South African foreign policy and (ii) further relevant details? CW187E' + >>> qn = '144. Mr D B Feldman (COPE-Gauteng) to ask the Minister of Defence and Military Veterans: Whether the deployment of the SA National Defence Force soldiers to the Central African Republic and the Democratic Republic of Congo is in line with our international policy with regard to (a) upholding international peace, (b) the promotion of constitutional democracy and (c) the respect for parliamentary democracy; if not, why not; if so, what are the (i) policies which underpin South African foreign policy and (ii) further relevant details? CW187E' >>> match = QuestionAnswerScraper.QUESTION_RE.match(qn) >>> match.groups() - (u'144. Mr D B Feldman (COPE-Gauteng) to ask the Minister of Defence and Military Veterans:', u'144', u'Mr D B Feldman', u'COPE-Gauteng', u'Minister of Defence and Military Veterans', None, u'', u'Whether the deployment of the SA National Defence Force soldiers to the Central African Republic and the Democratic Republic of Congo is in line with our international policy with regard to (a) upholding international peace, (b) the promotion of constitutional democracy and (c) the respect for parliamentary democracy; if not, why not; if so, what are the (i) policies which underpin South African foreign policy and (ii) further relevant details?', u'CW187E', u'C', u'W', u'187') + ('144. Mr D B Feldman (COPE-Gauteng) to ask the Minister of Defence and Military Veterans:', '144', 'Mr D B Feldman', 'COPE-Gauteng', 'Minister of Defence and Military Veterans', None, '', 'Whether the deployment of the SA National Defence Force soldiers to the Central African Republic and the Democratic Republic of Congo is in line with our international policy with regard to (a) upholding international peace, (b) the promotion of constitutional democracy and (c) the respect for parliamentary democracy; if not, why not; if so, what are the (i) policies which underpin South African foreign policy and (ii) further relevant details?', 'CW187E', 'C', 'W', '187') # Shows the need for \u2013 (en-dash) and / (in the date) in latter part of the intro - >>> qn = u'409. Mr M J R de Villiers (DA-WC) to ask the Minister of Public Works: [215] (Interdepartmental transfer \u2013 01/11) (a) What were the reasons for a cut back on the allocation for the Expanded Public Works Programme to municipalities in the 2013-14 financial year and (b) what effect will this have on (i) job creation and (ii) service delivery? CW603E' + >>> qn = '409. Mr M J R de Villiers (DA-WC) to ask the Minister of Public Works: [215] (Interdepartmental transfer \u2013 01/11) (a) What were the reasons for a cut back on the allocation for the Expanded Public Works Programme to municipalities in the 2013-14 financial year and (b) what effect will this have on (i) job creation and (ii) service delivery? CW603E' >>> match = QuestionAnswerScraper.QUESTION_RE.match(qn) >>> match.groups() - (u'409. Mr M J R de Villiers (DA-WC) to ask the Minister of Public Works: [215] (Interdepartmental transfer \u2013 01/11)', u'409', u'Mr M J R de Villiers', u'DA-WC', u'Minister of Public Works', None, u'', u'(a) What were the reasons for a cut back on the allocation for the Expanded Public Works Programme to municipalities in the 2013-14 financial year and (b) what effect will this have on (i) job creation and (ii) service delivery?', u'CW603E', u'C', u'W', u'603') + ('409. Mr M J R de Villiers (DA-WC) to ask the Minister of Public Works: [215] (Interdepartmental transfer \u2013 01/11)', '409', 'Mr M J R de Villiers', 'DA-WC', 'Minister of Public Works', None, '', '(a) What were the reasons for a cut back on the allocation for the Expanded Public Works Programme to municipalities in the 2013-14 financial year and (b) what effect will this have on (i) job creation and (ii) service delivery?', 'CW603E', 'C', 'W', '603') # Cope with missing close bracket - >>> qn = u'1517. Mr W P Doman (DA to ask the Minister of Cooperative Governance and Traditional Affairs: Which approximately 31 municipalities experienced service delivery protests as referred to in his reply to oral question 57 on 10 September 2009? NW1922E' + >>> qn = '1517. Mr W P Doman (DA to ask the Minister of Cooperative Governance and Traditional Affairs: Which approximately 31 municipalities experienced service delivery protests as referred to in his reply to oral question 57 on 10 September 2009? NW1922E' >>> match = QuestionAnswerScraper.QUESTION_RE.match(qn) >>> match.groups() - (u'1517. Mr W P Doman (DA to ask the Minister of Cooperative Governance and Traditional Affairs:', u'1517', u'Mr W P Doman', u'DA', u'Minister of Cooperative Governance and Traditional Affairs', None, u'', u'Which approximately 31 municipalities experienced service delivery protests as referred to in his reply to oral question 57 on 10 September 2009?', u'NW1922E', u'N', u'W', u'1922') + ('1517. Mr W P Doman (DA to ask the Minister of Cooperative Governance and Traditional Affairs:', '1517', 'Mr W P Doman', 'DA', 'Minister of Cooperative Governance and Traditional Affairs', None, '', 'Which approximately 31 municipalities experienced service delivery protests as referred to in his reply to oral question 57 on 10 September 2009?', 'NW1922E', 'N', 'W', '1922') # Check we cope with no space before party in parentheses - >>> qn = u'1569. Mr M Swart(DA) to ask the Minister of Finance: Test question? NW1975E' + >>> qn = '1569. Mr M Swart(DA) to ask the Minister of Finance: Test question? NW1975E' >>> match = QuestionAnswerScraper.QUESTION_RE.match(qn) >>> match.groups() - (u'1569. Mr M Swart(DA) to ask the Minister of Finance:', u'1569', u'Mr M Swart', u'DA', u'Minister of Finance', None, u'', u'Test question?', u'NW1975E', u'N', u'W', u'1975') + ('1569. Mr M Swart(DA) to ask the Minister of Finance:', '1569', 'Mr M Swart', 'DA', 'Minister of Finance', None, '', 'Test question?', 'NW1975E', 'N', 'W', '1975') # Check we cope with a dot after the askee instead of a colon. - >>> qn = u'1875. Mr G G Hill-Lewis (DA) to ask the Minister in the Presidency. National Planning Test question? NW2224E' + >>> qn = '1875. Mr G G Hill-Lewis (DA) to ask the Minister in the Presidency. National Planning Test question? NW2224E' >>> match = QuestionAnswerScraper.QUESTION_RE.match(qn) >>> match.groups() - (u'1875. Mr G G Hill-Lewis (DA) to ask the Minister in the Presidency. National Planning', u'1875', u'Mr G G Hill-Lewis', u'DA', u'Minister in the Presidency', None, u'', u'Test question?', u'NW2224E', u'N', u'W', u'2224') + ('1875. Mr G G Hill-Lewis (DA) to ask the Minister in the Presidency. National Planning', '1875', 'Mr G G Hill-Lewis', 'DA', 'Minister in the Presidency', None, '', 'Test question?', 'NW2224E', 'N', 'W', '2224') # Check we cope without a question number - >>> qn = u'Mr AM Matlhoko (EFF) to ask the Minister of Cooperative Governance and Traditional Affairs: Whether he has an immediate plan to assist?' + >>> qn = 'Mr AM Matlhoko (EFF) to ask the Minister of Cooperative Governance and Traditional Affairs: Whether he has an immediate plan to assist?' >>> match = QuestionAnswerScraper.QUESTION_RE.match(qn) >>> match.groups() - (u'Mr AM Matlhoko (EFF) to ask the Minister of Cooperative Governance and Traditional Affairs:', None, u'Mr AM Matlhoko', u'EFF', u'Minister of Cooperative Governance and Traditional Affairs', None, u'', u'Whether he has an immediate plan to assist?', None, None, None, None) + ('Mr AM Matlhoko (EFF) to ask the Minister of Cooperative Governance and Traditional Affairs:', None, 'Mr AM Matlhoko', 'EFF', 'Minister of Cooperative Governance and Traditional Affairs', None, '', 'Whether he has an immediate plan to assist?', None, None, None, None) """ questions = [] for match in self.QUESTION_RE.finditer(text): match_dict = match.groupdict() - answer_type = match_dict[u'answer_type'] + answer_type = match_dict['answer_type'] number1 = match_dict.pop('number1') if answer_type == 'O': if re.search('(?i)to ask the Deputy President', match_dict['intro']): - match_dict[u'dp_number'] = number1 + match_dict['dp_number'] = number1 elif re.search('(?i)to ask the President', match_dict['intro']): - match_dict[u'president_number'] = number1 + match_dict['president_number'] = number1 else: - match_dict[u'oral_number'] = number1 + match_dict['oral_number'] = number1 elif answer_type == 'W': - match_dict[u'written_number'] = number1 + match_dict['written_number'] = number1 - match_dict[u'translated'] = bool(match_dict[u'translated']) - match_dict[u'questionto'] = match_dict[u'questionto'].replace(':', '') + match_dict['translated'] = bool(match_dict['translated']) + match_dict['questionto'] = match_dict['questionto'].replace(':', '') match_dict['questionto'] = self.correct_minister_title(match_dict['questionto']) questions.append(match_dict)