Initial import

LICENSE

@@ -0,0 +1,16 @@
+Copyright (c) 2011 Andris Reinman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,23 @@
+# fetch
+
+Fetch url contents
+
+## Install
+
+    npm install fetch
+
+## Usage
+
+    var fetch = require("fetch");
+
+    fetch("http://www.google.com", function(error, response){
+        console.log(response.status);
+        console.log(response.headers);
+        console.log(response.body);
+    });
+
+See test.js for a complete example
+
+## License
+
+BSD

fetch.js

@@ -0,0 +1,160 @@
+var http = require("http"),
+    https = require("https"),
+    urllib = require("url"),
+    zlib = require('zlib');
+
+/*
+
+headers: {}
+payload: txt
+
+maxresponse (100kB)
+headers: {}
+maxredirects: 10
+
+*/
+
+module.exports = fetch;
+
+function fetch(url, options, callback){
+
+    if(!callback && typeof options == "function"){
+        callback = options;
+        options = undefined;
+    }
+
+    options = options || {};
+    if(typeof options.maxredirects != "number" && !(options.maxredirects instanceof Number)){
+        options.maxredirects = 10;
+    }
+
+    // todo: redirects
+    function go(url, i){
+        i = i || 0;
+
+        get(url, options, function(error, response){
+            if(error){
+                return callback(error);
+            }
+            if([301, 302].indexOf(response.status)>=0 && response.headers.location){
+                i++;
+                if(i>options.maxredirects){
+                    return callback(null, response);
+                }
+                go(response.headers.location, i);
+            }else{
+                return callback(null, response);
+            }
+        });
+    }
+
+    go(url);
+
+}
+
+function get(url, options, callback){
+
+    if(!callback && typeof options == "function"){
+        callback = options;
+        options = undefined;
+    }
+
+    options = options || {};
+
+    options.maxresponse = options.maxresponse || (100*1024); // 100kB
+
+    var urlparts = urllib.parse(url, false, true),
+        transport,
+        urloptions = {
+            host: urlparts.hostname,
+            port: urlparts.port,
+            path: urlparts.pathname + (urlparts.search || "") || "/",
+            method: options.payload?'POST':'GET'
+        };
+
+    if(!urloptions.port){
+        switch(urlparts.protocol){
+            case "https:":
+                urloptions.port = 443;
+                transport = https;
+                break;
+            case "https:":
+            default:
+                urloptions.port = 80;
+                transport = http;
+                break;
+        }
+    }
+
+    if(options.headers){
+        urloptions.headers = options.headers;
+    }else{
+        urloptions.headers = {};
+    }
+
+    if(!options.nocompress){
+        urloptions.headers['Accept-Encoding'] = 'gzip';
+    }
+
+    var req = transport.request(urloptions, function(res) {
+
+        var responseBody = new Buffer(0),
+            currentPart,
+            maxlen,
+            unpack,
+
+            receive = function(chunk){
+
+                if(responseBody.length + chunk.length>options.maxresponse){
+                    maxlen = options.maxresponse - responseBody.length;
+                }else{
+                    maxlen = chunk.length;
+                }
+                if(maxlen<=0)return;
+
+                currentPart = new Buffer(responseBody.length + maxlen);
+                responseBody.copy(currentPart);
+                chunk.copy(currentPart, responseBody.length, 0, maxlen);
+                responseBody = currentPart;
+            },
+
+            end = function(){
+                callback(null, {
+                    status: res.statusCode,
+                    headers: res.headers,
+                    body: responseBody
+                });
+            }
+
+        if(res.headers['content-encoding']){
+            switch(res.headers['content-encoding'].toLowerCase().trim()){
+                case "gzip":
+                    unpack = zlib.createGunzip();
+                    unpack.on("data", receive);
+                    unpack.on("error", callback);
+                    unpack.on("end", end);
+                    res.pipe(unpack);
+                    return;
+                case "deflate":
+                    unpack = zlib.createInflateRaw();
+                    unpack.on("data", receive);
+                    unpack.on("error", callback);
+                    unpack.on("end", end);
+                    res.pipe(unpack);
+                    return;
+            }
+        }
+
+        res.on('data', receive);
+        res.on('end', end);
+    });
+
+    req.on('error', callback);
+
+    if(options.payload){
+        req.end(options.payload);
+    }else{
+        req.end();
+    }
+
+}

package.json

@@ -0,0 +1,29 @@
+{
+    "name": "fetch",
+    "description": "Fetch URL contents",
+    "version": "0.1.0",
+    "author" : "Andris Reinman",
+    "maintainers":[
+        {
+            "name":"andris",
+            "email":"[email protected]"
+        }
+    ],
+    "homepage": "http://github.com/andris9/fetch",
+    "repository" : {
+        "type" : "git",
+        "url" : "http://github.com/andris9/fetch.git"
+    },
+    "main" : "./fetch",
+    "licenses" : [
+        {
+            "type": "MIT",
+            "url": "http://github.com/andris9/fetch/blob/master/LICENSE"
+        }
+    ],
+    "dependencies": {
+        "mimelib-noiconv":"*"
+    },
+    "engine": [ "node >=0.5.0" ],
+    "keywords": ["url"]
+}

test.js

@@ -0,0 +1,7 @@
+var fetch = require("./fetch");
+
+fetch("https://www.google.com", function(error, contents){
+    console.log(error || contents);
+    console.log(contents.body.toString("utf-8"))
+    console.log(contents.body.toString("utf-8").length)
+});