Big Data
John Samuel
CPE Lyon
Année: 2020-2021
Courriel: john(dot)samuel(at)cpe(dot)fr
John Samuel
CPE Lyon
Année: 2020-2021
Courriel: john(dot)samuel(at)cpe(dot)fr
from urllib import request
response = request.urlopen("https://en.wikipedia.org/wiki/Main_Page")
html = response.read()
from urllib import request
from lxml import html
document = html.parse(request.urlopen("https://en.wikipedia.org/wiki/Main_Page"))
for link in document.iter("a"):
if(link.get("href") is not None):
print(link.base_url+link.get("href"))
import requests
url = "https://api.github.com/users/johnsamuelwrites"
response = requests.get(url)
print(response.json())
import requests
url = "https://api.github.com/users/johnsamuelwrites/repos"
response = requests.get(url)
print(response.json())
from SPARQLWrapper import SPARQLWrapper, JSON
sparql = SPARQLWrapper("http://query.wikidata.org/sparql")
sparql.setQuery("""
SELECT ?item WHERE {
?item wdt:P31 wd:Q9143;
}
LIMIT 10
""")
sparql.setReturnFormat(JSON)
results = sparql.query().convert()
for result in results["results"]["bindings"]:
print(result)
 - Wikidata.png)
.png)
La gestion des connaissances clients
$ head /home/john/Downloads/query.csv
itemLabel,year
Amiga E,1993
Embarcadero Delphi,1995
Sather,1990
Microsoft Small Basic,2008
Squeak,1996
AutoIt,1999
Eiffel,1985
Eiffel,1986
Kent Recursive Calculator,1981
$ export HADOOP_HOME="..."
$ ./hive
hive> set hive.metastore.warehouse.dir=${env:HOME}/hive/warehouse;
$./hive
hive> set hive.metastore.warehouse.dir=${env:HOME}/hive/warehouse;
hive> create database mydb;
hive> use mydb;
$./hive
hive> use mydb;
hive> CREATE TABLE IF NOT EXISTS
proglang (name String, year int)
COMMENT "Programming Languages"
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
hive> LOAD DATA LOCAL INPATH '/home/john/Downloads/query.csv'
OVERWRITE INTO TABLE proglang;
$./hive
hive> SELECT * from proglang;
hive> SELECT * from proglang where year > 1980;
$./hive
hive> DELETE from proglang where year=1980;
FAILED: SemanticException [Error 10294]: Attempt to do update
or delete using transaction manager that does not support these operations.
$./hive
hive> use mydb;
hive> CREATE TABLE IF NOT EXISTS
proglangorc (name String, year int)
COMMENT "Programming Languages"
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
LINES TERMINATED BY '\n'
STORED AS ORC;
hive> LOAD DATA LOCAL INPATH '/home/john/Downloads/query.csv'
OVERWRITE INTO TABLE proglangorc;
FAILED: SemanticException Unable to load data to destination table.
Error: The file that you are trying to load does not match
the file format of the destination table.
$./hive
hive> insert overwrite table proglangorc select * from proglang;
hive> DELETE from proglangorc where year=1980;
FAILED: SemanticException [Error 10297]: Attempt to do update
or delete on table mydb.proglangorc that is not transactional
hive> ALTER TABLE proglangorc set TBLPROPERTIES ('transactional'='true') ;
hive> DELETE from proglangorc where year=1980;
hive> SELECT count(*) from proglangorc;
hive> SELECT count(*) from proglangorc where year=1980;
$./pyspark
>>> lines = sc.textFile("/home/john/Downloads/query.csv")
>>> lineLengths = lines.map(lambda s: len(s))
>>> totalLength = lineLengths.reduce(lambda a, b: a + b)
>>> print(totalLength)
$./pyspark
>>> lines = sc.textFile("/home/john/Downloads/query.csv")
>>> lineWordCount = lines.map(lambda s: len(s.split()))
>>> totalWords = lineWordCount.reduce(lambda a, b: a + b)
>>> print(totalWords)
$ export SPARK_HOME='.../spark/spark-x.x.x-bin-hadoopx.x/bin
$ export PYSPARK_PYTHON=/usr/bin/python3
$ export PYSPARK_DRIVER_PYTHON=jupyter
$ export PYSPARK_DRIVER_PYTHON_OPTS='notebook'
$ ./pyspark
.jpg)
.png)
