First commit

Author: nadjib

.classpath

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry kind="src" path="src"/>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+	<classpathentry kind="lib" path="lib/jsoup-1.8.3.jar"/>
+	<classpathentry kind="lib" path="lib/java-json.jar"/>
+	<classpathentry kind="output" path="bin"/>
+</classpath>

.gitignore

@@ -0,0 +1,50 @@
+.metadata
+bin/
+tmp/
+*.tmp
+*.bak
+*.swp
+*~.nib
+local.properties
+.settings/
+.loadpath
+.recommenders
+
+# Eclipse Core
+# .project
+
+# External tool builders
+.externalToolBuilders/
+
+# Locally stored "Eclipse launch configurations"
+*.launch
+
+# PyDev specific (Python IDE for Eclipse)
+*.pydevproject
+
+# CDT-specific (C/C++ Development Tooling)
+.cproject
+
+# JDT-specific (Eclipse Java Development Tools)
+# .classpath
+
+# Java annotation processor (APT)
+.factorypath
+
+# PDT-specific (PHP Development Tools)
+.buildpath
+
+# sbteclipse plugin
+.target
+
+# Tern plugin
+.tern-project
+
+# TeXlipse plugin
+.texlipse
+
+# STS (Spring Tool Suite)
+.springBeans
+
+# Code Recommenders
+.recommenders/

.project

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>crawler_ted_articles</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.jdt.core.javabuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.jdt.core.javanature</nature>
+	</natures>
+</projectDescription>

README.md

@@ -0,0 +1 @@
+Crawler TED articles

corpus/README.md

@@ -0,0 +1 @@
+This directory will contain the files of articles

lib/java-json.jar

@@ -0,0 +1,62 @@
+package com.nadjib.crawler.ted;
+import java.io.FileWriter;
+import java.util.ArrayList;
+
+import org.json.JSONObject;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.select.Elements;
+
+// https://github.com/jhy/jsoup
+// http://jsoup.org/
+// http://www.java2s.com/Code/Jar/j/Downloadjavajsonjar.htm
+
+public class ListURLs {
+
+	public static final String ROOT_URL = "https://www.ted.com";
+
+	public static final int NB_PAGES = 5;
+	// TODO Add all list of languages
+	public static final String LANG = "AR";
+
+	public static void main(String[] args) {
+		try {
+			System.out.println("Begin");
+
+			ArrayList<String> urlsArray = new ArrayList<String>();
+
+			for (int i = 1; i <= NB_PAGES; i++) {
+
+				System.out.println("Download page N° " + i + " ...");
+
+				Document doc = Jsoup.connect(ROOT_URL + "/talks?language=" + LANG + "&page=" + i).get();
+				Elements links = doc.select("[href~=^(?i)/talks/.*=" + LANG + "]");
+
+				for (int j = 1; j < links.size(); j++) {
+					String link = links.get(j).attr("href");
+					if (link != null) {
+						link = link.replaceFirst("[\\?|&].*", "");
+
+						if (!urlsArray.contains(link))
+							urlsArray.add(link);
+					}
+				}
+			}
+
+			JSONObject jsonObject = new JSONObject();
+			jsonObject.put("root_url", ROOT_URL);
+
+			jsonObject.put("child_urls", urlsArray);
+
+			FileWriter file = new FileWriter("corpus/_urls.json");
+			file.write(jsonObject.toString(4));
+
+			file.close();
+
+			System.out.println("Finish.");
+
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+	}
+}