Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Port importer to new versions of dependencies #10

Open
wants to merge 8 commits into
base: master
Choose a base branch
from
Open

Port importer to new versions of dependencies #10

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
9f9ab8a
Port importer to new versions of dependencies
Sep 20, 2017
ababaab
Fix doUntil test
Sep 20, 2017
17e311e
Don't increment count twice
Sep 21, 2017
9d90b60
Update world-universities.csv from univ.cc
Sep 21, 2017
0ed76d1
Don't write html entities to CSV
Sep 21, 2017
bb0b2fe
GB universities are listed under UK
Sep 21, 2017
483c16e
Trim whitespace, resolves endSly/world-universities-csv#6
Sep 21, 2017
cfacb3b
Mention importer in README
Sep 21, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 42 additions & 30 deletions importer/import.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,43 +1,58 @@
var jsdom = require("jsdom")
, async = require("async")
, csv = require("csv")
, fs = require('fs')
, request = require('request')
, stringify = require("csv-stringify")
, countries = require('country-data').countries;

var count = 0;
const { JSDOM } = jsdom;

var lastPage;
let count;

function readPage(page, write, cb) {
jsdom.env(page, ["http://code.jquery.com/jquery.js"], function (err, window) {
count = 0;
var firstItem = window.$('ol li a')[0];
if (firstItem) {
var currentPage = firstItem.innerHTML;
if (currentPage == lastPage)
return cb();
let lastPage;

lastPage = currentPage;
function readPage(body, write, cb) {
const { document } = (new JSDOM(body)).window;

count = 0;

const firstItem = document.querySelector('ol li a');
if (firstItem) {
const currentPage = firstItem.innerHTML;
if (currentPage === lastPage) {
return cb();
}
lastPage = currentPage;
}

window.$('ol li a').each(function (i, el) {
write(el.innerHTML, window.$(el).attr('href'));
++count;
});
cb();
});
const allItems = document.querySelectorAll('ol li a');

for (count = 0; count < allItems.length; count++) {
write(allItems[count].innerHTML, allItems[count].href);
count++;
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You are incrementing count twice

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

woops! fixed

}

cb();
}

var output = csv().to("world-universities.csv");
const fileStream = fs.createWriteStream('world-universities.csv');
const output = stringify();
output.on('readable', function() {
while(row = output.read()){
fileStream.write(row);
}
});

function loadList(dom, country, cb) {
var total = 0;
var start = 1;
let total = 0;
let start = 1;
process.stdout.write("["+country+"] ");
async.doUntil(function(cb) {
var page = "http://univ.cc/search.php?dom=" + dom + "&key=&start=" + start;
readPage(page, function (name, url) {
output.write([country, name, url]);
}, cb);
request("http://univ.cc/search.php?dom=" + dom + "&key=&start=" + start, function (error, response, body) {
readPage(body, function (name, url) {
output.write([country, name, url]);
}, cb);
});

}, function() {
start += 50;
Expand All @@ -51,17 +66,14 @@ function loadList(dom, country, cb) {
});
}

var countriesCodes = Object.keys(countries);
const countriesCodes = Object.keys(countries);

async.eachSeries(countriesCodes, function(country, cb) {
if (country.length != 2)
if (country.length !== 2)
return cb();

var dom = country == "US" ? "edu" : country;
loadList(dom.toLowerCase(), country, cb);
}, function() {
output.end();
});



Loading