Switch to using requests for http requests

It's a lot cleaner API than urllib2, and will be easier once we port
version.

Hopefully this doesn't break something. Probably it does break encoding
somewhere, because py2.

6 files changed, 43 insertions, 50 deletions

diff --git a/postgresqleu/adyen/management/commands/process_adyen_reports.py b/postgresqleu/adyen/management/commands/process_adyen_reports.py
index 698adc51..45728bc9 100644
--- a/postgresqleu/adyen/management/commands/process_adyen_reports.py
+++ b/postgresqleu/adyen/management/commands/process_adyen_reports.py
@@ -9,8 +9,9 @@ from django.conf import settings
 
 import re
 import csv
-import urllib2
 import StringIO
+import requests
+from requests.auth import HTTPBasicAuth
 from base64 import standard_b64encode
 from datetime import datetime, date
 from decimal import Decimal
@@ -43,18 +44,15 @@ class Command(BaseCommand):
                 with transaction.atomic():
                     if self.verbose:
                         self.stdout.write("Downloading {0}".format(report.url))
-                    req = urllib2.Request(report.url)
-                    req.add_header('Authorization', 'Basic %s' % (
-                        standard_b64encode('%s:%s' % (settings.ADYEN_REPORT_USER, settings.ADYEN_REPORT_PASSWORD)),
-                        ))
-                    u = urllib2.urlopen(req)
-                    resp = u.read()
-                    u.close()
-                    if len(resp) == 0:
+                    resp = requests.get(report.url,
+                                        auth=HTTPBasicAuth(settings.ADYEN_REPORT_USER, settings.ADYEN_REPORT_PASSWORD))
+                    if resp.status_code != 200:
+                        self.stderr.write("Downloaded report {0} and got status code {1}. Not storing, will try again.".format(report.url, resp.status_code))
+                    elif len(resp.text) == 0:
                         self.stderr.write("Downloaded report {0} and got zero bytes (no header). Not storing, will try again.".format(report.url))
                     else:
                         report.downloadedat = datetime.now()
-                        report.contents = resp
+                        report.contents = resp.text
                         report.save()
                         AdyenLog(message='Downloaded report {0}'.format(report.url), error=False).save()
             except Exception as ex:
diff --git a/postgresqleu/adyen/util.py b/postgresqleu/adyen/util.py
index 4b4b027c..53579c0a 100644
--- a/postgresqleu/adyen/util.py
+++ b/postgresqleu/adyen/util.py
@@ -5,8 +5,8 @@ from django.db import transaction
 from datetime import datetime, date
 from decimal import Decimal
 
-import json
-import urllib2
+import requests
+from requests.auth import HTTPBasicAuth
 from base64 import standard_b64encode
 
 from postgresqleu.mailqueue.util import send_simple_mail
@@ -409,19 +409,14 @@ class AdyenAPI(object):
                 e))
 
     def _api_call(self, apiurl, apiparam, okresponse):
-        apijson = json.dumps(apiparam)
-
-        req = urllib2.Request("{0}{1}".format(settings.ADYEN_APIBASEURL, apiurl))
-
-        req.add_header('Authorization', 'Basic {0}'.format(
-            standard_b64encode('{0}:{1}'.format(settings.ADYEN_WS_USER, settings.ADYEN_WS_PASSWORD))))
-        req.add_header('Content-type', 'application/json')
-        u = urllib2.urlopen(req, apijson)
-        resp = u.read()
-        if u.getcode() != 200:
-            raise Exception("http response code {0}".format(u.getcode()))
-        u.close()
-        r = json.loads(resp)
+        resp = requests.post("{0}{1}".format(settings.ADYEN_APIBASEURL, apiurl),
+                             auth=HTTPBasicAuth(settings.ADYEN_WS_USER, settings.ADYEN_WS_PASSWORD),
+                             json=apiparam,
+        )
+        if resp.status_code != 200:
+            raise Exception("http response code {0}".format(resp.status_code))
+
+        r = resp.json()
 
         if r['response'] != okresponse:
             raise Exception("response returned: {0}".format(r['response']))
diff --git a/postgresqleu/auth.py b/postgresqleu/auth.py
index b060d282..2509f941 100644
--- a/postgresqleu/auth.py
+++ b/postgresqleu/auth.py
@@ -30,6 +30,7 @@ import json
 import socket
 import urlparse
 import urllib
+import requests
 from Crypto.Cipher import AES
 from Crypto.Hash import SHA
 from Crypto import Random
@@ -191,12 +192,13 @@ def user_search(searchterm=None, userid=None):
     else:
         q = {'s': searchterm}
 
-    u = urllib.urlopen('%ssearch/?%s' % (
-        settings.PGAUTH_REDIRECT,
-        urllib.urlencode(q),
-        ))
-    (ivs, datas) = u.read().split('&')
-    u.close()
+    r = requests.get('{0}search/'.format(settings.PGAUTH_REDIRECT),
+                     params=q,
+    )
+    if r.status_code != 200:
+        return []
+
+    (ivs, datas) = r.text.encode('utf8').split('&')
 
     # Decryption time
     decryptor = AES.new(base64.b64decode(settings.PGAUTH_KEY),
diff --git a/postgresqleu/paypal/util.py b/postgresqleu/paypal/util.py
index 7c7b42bb..66ebb0fe 100644
--- a/postgresqleu/paypal/util.py
+++ b/postgresqleu/paypal/util.py
@@ -1,7 +1,6 @@
 from django.conf import settings
 
-import urllib2
-from urllib import urlencode
+import requests
 from urlparse import parse_qs
 from decimal import Decimal
 import itertools
@@ -23,8 +22,12 @@ class PaypalAPI(object):
     def _api_call(self, command, params):
         params.update(self.accessparam)
         params['METHOD'] = command
-        resp = urllib2.urlopen(self.API_ENDPOINT, urlencode(params)).read()
-        q = parse_qs(resp)
+
+        resp = requests.post(self.API_ENDPOINT, data=params)
+        if resp.status_code != 200:
+            raise Exception("API error from paypal, status {0}".format(resp.status_code))
+
+        q = parse_qs(resp.text)
         if q['ACK'][0] != 'Success':
             raise Exception("API error from paypal: {0}/{1}".format(q['L_SHORTMESSAGE0'][0], q['L_LONGMESSAGE0'][0]))
         return q
diff --git a/postgresqleu/paypal/views.py b/postgresqleu/paypal/views.py
index f7bace57..de081eb2 100644
--- a/postgresqleu/paypal/views.py
+++ b/postgresqleu/paypal/views.py
@@ -5,7 +5,6 @@ from django.conf import settings
 
 from datetime import datetime, date
 from decimal import Decimal
-import urllib2
 from urllib import urlencode, unquote_plus
 
 from postgresqleu.invoices.util import InvoiceManager
@@ -62,9 +61,10 @@ def paypal_return_handler(request):
             'tx': tx,
             'at': settings.PAYPAL_PDT_TOKEN,
             }
-        u = urllib2.urlopen(settings.PAYPAL_BASEURL, urlencode(params))
-        r = u.read()
-        u.close()
+        resp = requests.post(settings.PAYPAL_BASEURL, data=params).decode('utf8')
+        if resp.status_code != 200:
+            raise Exception("status code {0}".format(resp.status_code))
+        r = resp.text
     except Exception as ex:
         # Failed to talk to paypal somehow. It should be ok to retry.
         return paypal_error('Failed to verify status with paypal: %s' % ex)
diff --git a/postgresqleu/trustlypayment/api.py b/postgresqleu/trustlypayment/api.py
index 9c8e268b..22fe67f3 100644
--- a/postgresqleu/trustlypayment/api.py
+++ b/postgresqleu/trustlypayment/api.py
@@ -6,7 +6,7 @@ from Crypto.Signature import PKCS1_v1_5
 from Crypto.Hash import SHA
 from Crypto.PublicKey import RSA
 import base64
-import urllib2
+import requests
 
 
 class TrustlyException(Exception):
@@ -102,16 +102,11 @@ class TrustlyWrapper(object):
             'params': params,
             'version': '1.1',
         }
-        apijson = json.dumps(p)
-
-        req = urllib2.Request(self.apibase)
-        req.add_header('Content-type', 'application/json')
-        u = urllib2.urlopen(req, apijson)
-        resp = u.read()
-        if u.getcode() != 200:
-            raise TrustlyException("bad http response code {0}".format(u.getcode()))
-        u.close()
-        r = json.loads(resp)
+
+        resp = requests.post(self.apibase, json=p)
+        if resp.status_code != 200:
+            raise TrustlyException("bad http response code {0}".format(resp.status_code))
+        r = resp.json()
         if 'error' in r:
             # XXX log and raise generic exception!
             raise TrustlyException(r['error']['message'])