Skip to content

Commit 58bc8b9

Browse files
mikel-egana-arangurenmikel-egana-aranguren
committed
Java to clean data
Programa de Adrian para limpiar datos, que dan pena la verdad
1 parent 17936aa commit 58bc8b9

File tree

4 files changed

+295
-0
lines changed

4 files changed

+295
-0
lines changed

‎loadparkingdata/pom.xml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
2+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
3+
<modelVersion>4.0.0</modelVersion>
4+
5+
<groupId>eus.eurohelp.loader</groupId>
6+
<artifactId>loadparkingdata</artifactId>
7+
<version>0.0.1-SNAPSHOT</version>
8+
<packaging>jar</packaging>
9+
10+
<name>loadparkingdata</name>
11+
<url>http://maven.apache.org</url>
12+
13+
<properties>
14+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
15+
</properties>
16+
17+
<dependencies>
18+
<dependency>
19+
<groupId>junit</groupId>
20+
<artifactId>junit</artifactId>
21+
<version>3.8.1</version>
22+
<scope>test</scope>
23+
</dependency>
24+
<!-- https://mvnrepository.com/artifact/com.googlecode.json-simple/json-simple -->
25+
<dependency>
26+
<groupId>com.googlecode.json-simple</groupId>
27+
<artifactId>json-simple</artifactId>
28+
<version>1.1.1</version>
29+
</dependency>
30+
<!-- https://mvnrepository.com/artifact/commons-io/commons-io -->
31+
<dependency>
32+
<groupId>commons-io</groupId>
33+
<artifactId>commons-io</artifactId>
34+
<version>2.5</version>
35+
</dependency>
36+
</dependencies>
37+
</project>

‎loadparkingdata/src/main/java/eus/eurohelp/loader/loadparkingdata/App.java‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
package eus.eurohelp.loader.loadparkingdata;
2+
3+
import java.io.File;
4+
import java.io.IOException;
5+
import java.nio.charset.StandardCharsets;
6+
import java.util.List;
7+
import java.util.regex.Matcher;
8+
import java.util.regex.Pattern;
9+
10+
import org.apache.commons.io.FileUtils;
11+
import org.json.simple.JSONObject;
12+
13+
public class App {
14+
public static void main(String[] args) {
15+
System.out.println("Inicio LoadDataParkings load");
16+
// String downloadURL =
17+
// "http://www.donostia.eus/info/ciudadano/camaras_trafico.nsf/dameParkings?OpenAgent&idioma=cas";
18+
String downloadURL = args[0];
19+
String file_name = args[1];
20+
String downloadPath = args[2]; // "ODEfiles_mobility_parking";
21+
String downloadFileName = args[3]; // "parkings.json";
22+
String charSet = StandardCharsets.ISO_8859_1.name();
23+
24+
FileManagement fm = new FileManagement(downloadURL, downloadPath, downloadFileName, charSet);
25+
26+
// Descarga de archivo
27+
fm.downloadFile();
28+
29+
// Se obtiene los objetos incidencias
30+
List<JSONObject> listParkings = fm.getContentOtherMobilityParking();
31+
32+
System.out.println(
33+
"Nombre,PlazasRotatorias,Tipo,PlazasResidentes,PlazasTotales,PlazasResidentesLibres,PlazasRotatoriasLibres,Precios,Latitud,Longitud");
34+
String csv = "Nombre,PlazasRotatorias,Tipo,PlazasResidentes,PlazasTotales,PlazasResidentesLibres,PlazasRotatoriasLibres,Precios,Latitud,Longitud\n";
35+
36+
// System.out.println(
37+
// "Nombre,PlazasRotatorias,Tipo,PlazasResidentes,PlazasTotales,PlazasResidentesLibres,Precios,Latitud,Longitud");
38+
// String csv = "Nombre,PlazasRotatorias,Tipo,PlazasResidentes,PlazasTotales,PlazasResidentesLibres,Precios,Latitud,Longitud\n";
39+
40+
for (JSONObject parking : listParkings) {
41+
if (!(parking.get("Nombre").toString().contains("Estac"))) {
42+
System.out.print(parking.getOrDefault("Nombre", "-") + ",");
43+
csv += parking.getOrDefault("Nombre", "-") + ",";
44+
System.out.print(parking.getOrDefault("PlazasRotatorias", "-") + ",");
45+
csv += parking.getOrDefault("PlazasRotatorias", "-") + ",";
46+
System.out.print(parking.getOrDefault("Tipo", "-") + ",");
47+
csv += parking.getOrDefault("Tipo", "-") + ",";
48+
System.out.print(parking.getOrDefault("PlazasResidentes", "-") + ",");
49+
csv += parking.getOrDefault("PlazasResidentes", "-") + ",";
50+
System.out.print(parking.getOrDefault("PlazasTotales", "-") + ",");
51+
csv += parking.getOrDefault("PlazasTotales", "-") + ",";
52+
System.out.print(parking.getOrDefault("PlazasResidentesLibres", "-") + ",");
53+
csv += parking.getOrDefault("PlazasResidentesLibres", "-") + ",";
54+
System.out.print(parking.getOrDefault("PlazasRotatoriasLibres", "0") + ",");
55+
csv += parking.getOrDefault("PlazasRotatoriasLibres", "0") + ",";
56+
57+
Pattern pattern = Pattern.compile("\\d{1,3}(?![0-9%]{1,3})");
58+
Matcher matcher = pattern.matcher((String) parking.getOrDefault("Datos", "-"));
59+
String rotativePlacesFree = "0";
60+
if (matcher.find()) {
61+
rotativePlacesFree = matcher.group();
62+
System.out.print(rotativePlacesFree + ",");
63+
csv += rotativePlacesFree + ",";
64+
}
65+
66+
String preciosParkings = (String) parking.getOrDefault("Precios", "-");
67+
68+
preciosParkings = preciosParkings.replaceAll(",", ".");
69+
preciosParkings = preciosParkings.replaceAll("Precios", "");
70+
preciosParkings = preciosParkings.replaceAll(
71+
"<br/>|<strong>|</strong>|<table>|</table>|<tr>|</tr>|<td>|</td>|<td class=\"azul\">", "");
72+
preciosParkings = preciosParkings.replaceAll(" &#8364;", ";");
73+
74+
if (preciosParkings.isEmpty()) {
75+
System.out.print("-,");
76+
csv += "-,";
77+
} else {
78+
System.out.print(preciosParkings);
79+
csv += preciosParkings + ",";
80+
}
81+
82+
System.out.print(parking.getOrDefault("Latitud", "-") + ",");
83+
csv += parking.getOrDefault("Latitud", "-") + ",";
84+
System.out.print(parking.getOrDefault("Longitud", "-"));
85+
csv += parking.getOrDefault("Longitud", "-") + "\n";
86+
87+
System.out.println();
88+
}
89+
}
90+
91+
File file = new File(file_name);
92+
try {
93+
FileUtils.writeStringToFile(file, csv);
94+
} catch (IOException e) {
95+
e.printStackTrace();
96+
}
97+
}
98+
}

‎loadparkingdata/src/main/java/eus/eurohelp/loader/loadparkingdata/FileManagement.java‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
package eus.eurohelp.loader.loadparkingdata;
2+
3+
import java.io.File;
4+
import java.io.FileInputStream;
5+
import java.io.FileNotFoundException;
6+
import java.io.IOException;
7+
import java.io.InputStream;
8+
import java.io.InputStreamReader;
9+
import java.io.Reader;
10+
import java.net.MalformedURLException;
11+
import java.net.URL;
12+
import java.util.ArrayList;
13+
import java.util.List;
14+
15+
import org.apache.commons.io.FileUtils;
16+
import org.json.simple.JSONArray;
17+
import org.json.simple.JSONObject;
18+
import org.json.simple.parser.JSONParser;
19+
import org.json.simple.parser.ParseException;
20+
21+
/**
22+
* Clase para gestionar los datos en crudo de varias fuentes de datos
23+
*
24+
* @author acarenas Created at 21 de oct. de 2016
25+
*/
26+
public class FileManagement {
27+
28+
private String downloadURL;
29+
private String downloadPath;
30+
private String downloadFileName;
31+
private String unzipPath;
32+
private String differentLinesPathNew;
33+
private String differentLinesPathOld;
34+
private String charSet;
35+
36+
/**
37+
*
38+
* Constructor for FileManagement
39+
*
40+
*/
41+
public FileManagement(String downloadURL, String downloadPath, String unzipPath, String differentLinesPathNew,
42+
String differentLinesPathOld, String downloadFileName, String charSet) {
43+
this.downloadURL = downloadURL;
44+
this.downloadPath = downloadPath;
45+
this.unzipPath = unzipPath;
46+
this.differentLinesPathNew = differentLinesPathNew;
47+
this.differentLinesPathOld = differentLinesPathOld;
48+
this.downloadFileName = downloadFileName;
49+
this.charSet = charSet;
50+
}
51+
52+
/**
53+
*
54+
* Constructor for FileManagement ODE mobility traffic
55+
*
56+
*/
57+
public FileManagement(String downloadURL, String downloadPath, String downloadFileName, String charSet) {
58+
this.downloadURL = downloadURL;
59+
this.downloadPath = downloadPath;
60+
this.downloadFileName = downloadFileName;
61+
this.charSet = charSet;
62+
}
63+
64+
/**
65+
* Método que obtiene los datos de los aparcamientos a partir de la fuente
66+
* de datos de donostia Debido a CLRFs en archivo se opta por JSONsimple, el
67+
* json es pequeño por lo que no sería muy ineficiente remplazar esos
68+
* caracteres y usar jackson.
69+
*
70+
* @return List<GeolocationIncidence>
71+
*
72+
* @author acarenas
73+
*/
74+
public List<JSONObject> getContentOtherMobilityParking() {
75+
76+
List<JSONObject> listParkings = new ArrayList<JSONObject>();
77+
JSONParser parser = new JSONParser();
78+
JSONArray jsonArray;
79+
try {
80+
File file = new File(this.downloadPath + File.separator + this.downloadFileName);
81+
InputStream is = new FileInputStream(file);
82+
Reader reader = new InputStreamReader(is, this.charSet);
83+
jsonArray = (JSONArray) parser.parse(reader);
84+
for (int i = 0; i < jsonArray.size(); i++) {
85+
listParkings.add((JSONObject) jsonArray.get(i));
86+
}
87+
} catch (FileNotFoundException e) {
88+
System.out.println("El archivo " + this.downloadPath + File.separator + this.downloadFileName + " no existe");
89+
} catch (IOException e) {
90+
System.out.println("El archivo " + this.downloadPath + File.separator + this.downloadFileName
91+
+ " no se ha podido acceder.");
92+
} catch (ParseException e) {
93+
System.out.println("El archivo JSON no se ha podido parsear.");
94+
}
95+
96+
return listParkings;
97+
}
98+
99+
/**
100+
* Descarga un archivo en una ruta
101+
*
102+
* @param fileUrl
103+
* String
104+
* @param fileName
105+
* String
106+
*
107+
* @author acarenas
108+
*/
109+
public void downloadFile() {
110+
try {
111+
FileUtils.copyURLToFile(new URL(this.downloadURL),
112+
new File(this.downloadPath + File.separator + this.downloadFileName));
113+
System.out.println(this.downloadURL);
114+
System.out.println(this.downloadPath + File.separator + this.downloadFileName);
115+
} catch (MalformedURLException e) {
116+
System.out.println("La URL no es correcta");
117+
} catch (IOException e) {
118+
System.out.println("El archivo no se ha descargado correctamente");
119+
}
120+
}
121+
122+
}

‎loadparkingdata/src/test/java/eus/eurohelp/loader/loadparkingdata/AppTest.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
package eus.eurohelp.loader.loadparkingdata;
2+
3+
import junit.framework.Test;
4+
import junit.framework.TestCase;
5+
import junit.framework.TestSuite;
6+
7+
/**
8+
* Unit test for simple App.
9+
*/
10+
public class AppTest
11+
extends TestCase
12+
{
13+
/**
14+
* Create the test case
15+
*
16+
* @param testName name of the test case
17+
*/
18+
public AppTest( String testName )
19+
{
20+
super( testName );
21+
}
22+
23+
/**
24+
* @return the suite of tests being tested
25+
*/
26+
public static Test suite()
27+
{
28+
return new TestSuite( AppTest.class );
29+
}
30+
31+
/**
32+
* Rigourous Test :-)
33+
*/
34+
public void testApp()
35+
{
36+
assertTrue( true );
37+
}
38+
}

0 commit comments

Comments
 (0)