Big Data
John Samuel
CPE Lyon
Année: 2017-2018
Courriel: john(dot)samuel(at)cpe(dot)fr
John Samuel
CPE Lyon
Année: 2017-2018
Courriel: john(dot)samuel(at)cpe(dot)fr
from urllib import request
response = request.urlopen("https://en.wikipedia.org/wiki/Main_Page")
html = response.read()
from urllib import request
from lxml import html
document = html.parse(request.urlopen("https://en.wikipedia.org/wiki/Main_Page"))
for link in document.iter("a"):
if(link.get("href") is not None):
print(link.base_url+link.get("href"))
import requests
url = "https://api.github.com/users/johnsamuelwrites"
response = requests.get(url)
print(response.json())
import requests
url = "https://api.github.com/users/johnsamuelwrites/repos"
response = requests.get(url)
print(response.json())
from SPARQLWrapper import SPARQLWrapper, JSON
sparql = SPARQLWrapper("http://query.wikidata.org/sparql")
sparql.setQuery("""
SELECT ?item WHERE {
?item wdt:P31 wd:Q9143;
}
LIMIT 10
""")
sparql.setReturnFormat(JSON)
results = sparql.query().convert()
for result in results["results"]["bindings"]:
print(result)
 - Wikidata.png)
.png)
$./hive
hive> set hive.metastore.warehouse.dir=${env:HOME}/hive/warehouse;
$./hive
hive> set hive.metastore.warehouse.dir=${env:HOME}/hive/warehouse;
hive> create database mydb;
hive> use mydb;
$./hive
hive> use mydb;
hive> CREATE TABLE IF NOT EXISTS proglang (name String, year int) COMMENT "Programming Languages" ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' STORED AS TEXTFILE;
hive> LOAD DATA LOCAL INPATH '/home/john/Downloads/query.csv' OVERWRITE INTO TABLE proglang;
$./hive
hive> SELECT * from proglang;
hive> SELECT * from proglang where year > 1980;
$./pyspark
>>> lines = sc.textFile("/home/john/Downloads/query.csv")
>>> lineLengths = lines.map(lambda s: len(s))
>>> totalLength = lineLengths.reduce(lambda a, b: a + b)
>>> print(totalLength)
.jpg)
.png)
