Page MenuHomeGitPull.it
Differential D42

refactor with new Query classes
ClosedPublic
Actions

Authored by valerio.bozzolan on Oct 12 2020, 08:06.
Tags
None
Referenced Files
F2913388: D42.1742357918.diff
Tue, Mar 18, 05:18
F2913387: D42.1742357902.diff
Tue, Mar 18, 05:18
F2913168: D42.1742331526.diff
Mon, Mar 17, 21:58
F2912206: D42.1742291595.diff
Mon, Mar 17, 10:53
F2912091: D42.1742288778.diff
Mon, Mar 17, 10:06
Subscribers
None

Details

Reviewers
None
Commits
R41:f56203a0ec0a: refactor with new Query classes

Diff Detail

Repository
R41 suckless-conference
Branch
master
Lint
No Lint Coverage
Unit
No Test Coverage
Build Status
Buildable 126
Build 178: arc lint + arc unit

Event Timeline

valerio.bozzolan requested review of this revision.Oct 12 2020, 08:06
valerio.bozzolan created this revision.
Harbormaster completed remote builds in B126: Diff 145.Oct 12 2020, 08:06
This revision was not accepted when it landed; it landed in state Needs Review.Oct 15 2020, 23:42
Closed by commit R41:f56203a0ec0a: refactor with new Query classes. · Explain Why
This revision was automatically updated to reflect the committed changes.
valerio.bozzolan added a commit: R41:f56203a0ec0a: refactor with new Query classes.

Revision Contents
Changeset List

Diff 145

View Options

includes/class-DailyEvents.php

package it.reyboz.bustorino.backend.gtfs; package it.reyboz.bustorino.backend.gtfs;
import android.content.Context; import android.content.Context;
import android.util.Log; import android.util.Log;
import androidx.annotation.NonNull; import androidx.annotation.NonNull;
import de.siegmar.fastcsv.reader.CloseableIterator;
import de.siegmar.fastcsv.reader.NamedCsvReader;
import de.siegmar.fastcsv.reader.NamedCsvRow;
import it.reyboz.bustorino.backend.Fetcher; import it.reyboz.bustorino.backend.Fetcher;
import it.reyboz.bustorino.backend.networkTools; import it.reyboz.bustorino.backend.networkTools;
import it.reyboz.bustorino.data.gtfs.CsvTableInserter; import it.reyboz.bustorino.data.gtfs.CsvTableInserter;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Attributes; import org.jsoup.nodes.Attributes;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.jsoup.select.Elements; import org.jsoup.select.Elements;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection; import java.net.HttpURLConnection;
import java.net.URL; import java.net.URL;
import java.text.ParseException; import java.text.ParseException;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.*; import java.util.*;
import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.zip.ZipEntry; import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipInputStream; import java.util.zip.ZipInputStream;
abstract public class GtfsDataParser { abstract public class GtfsDataParser {
public static final String GTFS_ADDRESS="https://www.gtt.to.it/open_data/gtt_gtfs.zip"; public static final String GTFS_ADDRESS="https://www.gtt.to.it/open_data/gtt_gtfs.zip";
public static final String GTFS_PAGE_ADDRESS="http://aperto.comune.torino.it/dataset/feed-gtfs-trasporti-gtt"; public static final String GTFS_PAGE_ADDRESS="http://aperto.comune.torino.it/dataset/feed-gtfs-trasporti-gtt";
private static final String DEBUG_TAG = "BusTO-GTFSDataParser"; private static final String DEBUG_TAG = "BusTO-GTFSDataParser";
▲ Show 20 Lines • Show All 103 Lines • ▼ Show 20 Lines public static Date getLastGTFSUpdateDate(AtomicReference<Fetcher.Result> res) {
} }
break; break;
} }
} }
res.set(Fetcher.Result.PARSER_ERROR); res.set(Fetcher.Result.PARSER_ERROR);
return finalDate; return finalDate;
} }
public static void readGtfsZipEntry(ZipEntry entry, ZipFile zipFile, Context con) throws IOException{
String tableName = entry.getName().split("\\.")[0].trim();
InputStream stream = zipFile.getInputStream(entry);
String s = String.format(Locale.ENGLISH, "Entry: %s len %d added",
entry.getName(),
entry.getSize()
);
Log.d(DEBUG_TAG, s);
final BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
GtfsDataParser.readCSVWithColumns(reader, tableName, con);
stream.close();
}
public static void readCSVWithColumns(BufferedReader reader, String tableName, Context con) throws IOException { public static void readCSVWithColumns(BufferedReader reader, String tableName, Context con) throws IOException {
//String[] elements; //String[] elements;
List<String> lineElements; List<String> lineElements;
String line; String line;
final String header = reader.readLine(); /*final String header = reader.readLine();
if (header == null){ if (header == null){
throw new IOException(); throw new IOException();
} }*/
//elements = header.split("\n")[0].split(","); //elements = header.split("\n")[0].split(",");
//System.out.println(Arrays.toString(elements)); //System.out.println(Arrays.toString(elements));
lineElements = readCsvLine(header); //lineElements = readCsvLine(header);
NamedCsvReader csvReader = NamedCsvReader.builder().build(reader);
CloseableIterator<NamedCsvRow> iterator = csvReader.iterator();
final HashMap<Integer,String> columnMap = new HashMap<>();
final CsvTableInserter inserter = new CsvTableInserter(tableName,con); final CsvTableInserter inserter = new CsvTableInserter(tableName,con);
/*final HashMap<Integer,String> columnMap = new HashMap<>();
for (int i=0; i< lineElements.size(); i++){ for (int i=0; i< lineElements.size(); i++){
//columnMap.put(i, fixStringIfItHasQuotes(elements[i].trim()) ); //columnMap.put(i, fixStringIfItHasQuotes(elements[i].trim()) );
columnMap.put(i, lineElements.get(i).trim() ); columnMap.put(i, lineElements.get(i).trim() );
} }
Log.d(DEBUG_TAG, "Columns for the file: "+columnMap); Log.d(DEBUG_TAG, "Columns for the file: "+columnMap);
boolean first = true; boolean first = true;
while((line = reader.readLine())!=null){ while((line = reader.readLine())!=null){
//there is a line of data //there is a line of data
//elements = line.split("\n")[0].split(","); //elements = line.split("\n")[0].split(",");
if(first) Log.d(DEBUG_TAG, "Element line: "+line); if(first) Log.d(DEBUG_TAG, "Element line: "+line);
lineElements = readCsvLine(line); lineElements = readCsvLine(line);
final Map<String,String> rowsMap = getColumnsAsString(lineElements.toArray(new String[0]), columnMap); final Map<String,String> rowsMap = getColumnsAsString(lineElements.toArray(new String[0]), columnMap);
if (first){ if (first){
Log.d(DEBUG_TAG, " in map:"+rowsMap); Log.d(DEBUG_TAG, " in map:"+rowsMap);
first=false; first=false;
} }
inserter.addElement(rowsMap); inserter.addElement(rowsMap);
}*/
int c = 0;
while (iterator.hasNext()){
final Map<String,String> rowsMap = iterator.next().getFields();
if (c < 1){
Log.d(DEBUG_TAG, " in map:"+rowsMap);
c++;
}
inserter.addElement(rowsMap);
} }
//commit data //commit data
inserter.insertDataInDatabase(); inserter.finishInsert();
} }
@NonNull @NonNull
private static Map<String,String> getColumnsAsString(@NonNull String[] lineElements, Map<Integer,String> colsIndices) private static Map<String,String> getColumnsAsString(@NonNull String[] lineElements, Map<Integer,String> colsIndices)
{ {
final HashMap<String,String> theMap = new HashMap<>(); final HashMap<String,String> theMap = new HashMap<>();
for(int l=0; l<lineElements.length; l++){ for(int l=0; l<lineElements.length; l++){
if(!colsIndices.containsKey(l)) if(!colsIndices.containsKey(l))
continue; continue;
▲ Show 20 Lines • Show All 57 Lines • Show Last 20 Lines
View Options

includes/class-QueryChapter.php

package it.reyboz.bustorino.backend.gtfs; package it.reyboz.bustorino.backend.gtfs;
import android.content.Context; import android.content.Context;
import android.util.Log; import android.util.Log;
import androidx.annotation.NonNull; import androidx.annotation.NonNull;
import de.siegmar.fastcsv.reader.CloseableIterator;
import de.siegmar.fastcsv.reader.NamedCsvReader;
import de.siegmar.fastcsv.reader.NamedCsvRow;
import it.reyboz.bustorino.backend.Fetcher; import it.reyboz.bustorino.backend.Fetcher;
import it.reyboz.bustorino.backend.networkTools; import it.reyboz.bustorino.backend.networkTools;
import it.reyboz.bustorino.data.gtfs.CsvTableInserter; import it.reyboz.bustorino.data.gtfs.CsvTableInserter;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Attributes; import org.jsoup.nodes.Attributes;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.jsoup.select.Elements; import org.jsoup.select.Elements;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection; import java.net.HttpURLConnection;
import java.net.URL; import java.net.URL;
import java.text.ParseException; import java.text.ParseException;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.*; import java.util.*;
import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.zip.ZipEntry; import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipInputStream; import java.util.zip.ZipInputStream;
abstract public class GtfsDataParser { abstract public class GtfsDataParser {
public static final String GTFS_ADDRESS="https://www.gtt.to.it/open_data/gtt_gtfs.zip"; public static final String GTFS_ADDRESS="https://www.gtt.to.it/open_data/gtt_gtfs.zip";
public static final String GTFS_PAGE_ADDRESS="http://aperto.comune.torino.it/dataset/feed-gtfs-trasporti-gtt"; public static final String GTFS_PAGE_ADDRESS="http://aperto.comune.torino.it/dataset/feed-gtfs-trasporti-gtt";
private static final String DEBUG_TAG = "BusTO-GTFSDataParser"; private static final String DEBUG_TAG = "BusTO-GTFSDataParser";
▲ Show 20 Lines • Show All 103 Lines • ▼ Show 20 Lines public static Date getLastGTFSUpdateDate(AtomicReference<Fetcher.Result> res) {
} }
break; break;
} }
} }
res.set(Fetcher.Result.PARSER_ERROR); res.set(Fetcher.Result.PARSER_ERROR);
return finalDate; return finalDate;
} }
public static void readGtfsZipEntry(ZipEntry entry, ZipFile zipFile, Context con) throws IOException{
String tableName = entry.getName().split("\\.")[0].trim();
InputStream stream = zipFile.getInputStream(entry);
String s = String.format(Locale.ENGLISH, "Entry: %s len %d added",
entry.getName(),
entry.getSize()
);
Log.d(DEBUG_TAG, s);
final BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
GtfsDataParser.readCSVWithColumns(reader, tableName, con);
stream.close();
}
public static void readCSVWithColumns(BufferedReader reader, String tableName, Context con) throws IOException { public static void readCSVWithColumns(BufferedReader reader, String tableName, Context con) throws IOException {
//String[] elements; //String[] elements;
List<String> lineElements; List<String> lineElements;
String line; String line;
final String header = reader.readLine(); /*final String header = reader.readLine();
if (header == null){ if (header == null){
throw new IOException(); throw new IOException();
} }*/
//elements = header.split("\n")[0].split(","); //elements = header.split("\n")[0].split(",");
//System.out.println(Arrays.toString(elements)); //System.out.println(Arrays.toString(elements));
lineElements = readCsvLine(header); //lineElements = readCsvLine(header);
NamedCsvReader csvReader = NamedCsvReader.builder().build(reader);
CloseableIterator<NamedCsvRow> iterator = csvReader.iterator();
final HashMap<Integer,String> columnMap = new HashMap<>();
final CsvTableInserter inserter = new CsvTableInserter(tableName,con); final CsvTableInserter inserter = new CsvTableInserter(tableName,con);
/*final HashMap<Integer,String> columnMap = new HashMap<>();
for (int i=0; i< lineElements.size(); i++){ for (int i=0; i< lineElements.size(); i++){
//columnMap.put(i, fixStringIfItHasQuotes(elements[i].trim()) ); //columnMap.put(i, fixStringIfItHasQuotes(elements[i].trim()) );
columnMap.put(i, lineElements.get(i).trim() ); columnMap.put(i, lineElements.get(i).trim() );
} }
Log.d(DEBUG_TAG, "Columns for the file: "+columnMap); Log.d(DEBUG_TAG, "Columns for the file: "+columnMap);
boolean first = true; boolean first = true;
while((line = reader.readLine())!=null){ while((line = reader.readLine())!=null){
//there is a line of data //there is a line of data
//elements = line.split("\n")[0].split(","); //elements = line.split("\n")[0].split(",");
if(first) Log.d(DEBUG_TAG, "Element line: "+line); if(first) Log.d(DEBUG_TAG, "Element line: "+line);
lineElements = readCsvLine(line); lineElements = readCsvLine(line);
final Map<String,String> rowsMap = getColumnsAsString(lineElements.toArray(new String[0]), columnMap); final Map<String,String> rowsMap = getColumnsAsString(lineElements.toArray(new String[0]), columnMap);
if (first){ if (first){
Log.d(DEBUG_TAG, " in map:"+rowsMap); Log.d(DEBUG_TAG, " in map:"+rowsMap);
first=false; first=false;
} }
inserter.addElement(rowsMap); inserter.addElement(rowsMap);
}*/
int c = 0;
while (iterator.hasNext()){
final Map<String,String> rowsMap = iterator.next().getFields();
if (c < 1){
Log.d(DEBUG_TAG, " in map:"+rowsMap);
c++;
}
inserter.addElement(rowsMap);
} }
//commit data //commit data
inserter.insertDataInDatabase(); inserter.finishInsert();
} }
@NonNull @NonNull
private static Map<String,String> getColumnsAsString(@NonNull String[] lineElements, Map<Integer,String> colsIndices) private static Map<String,String> getColumnsAsString(@NonNull String[] lineElements, Map<Integer,String> colsIndices)
{ {
final HashMap<String,String> theMap = new HashMap<>(); final HashMap<String,String> theMap = new HashMap<>();
for(int l=0; l<lineElements.length; l++){ for(int l=0; l<lineElements.length; l++){
if(!colsIndices.containsKey(l)) if(!colsIndices.containsKey(l))
continue; continue;
▲ Show 20 Lines • Show All 57 Lines • Show Last 20 Lines
View Options

includes/class-QueryEvent.php

package it.reyboz.bustorino.backend.gtfs; package it.reyboz.bustorino.backend.gtfs;
import android.content.Context; import android.content.Context;
import android.util.Log; import android.util.Log;
import androidx.annotation.NonNull; import androidx.annotation.NonNull;
import de.siegmar.fastcsv.reader.CloseableIterator;
import de.siegmar.fastcsv.reader.NamedCsvReader;
import de.siegmar.fastcsv.reader.NamedCsvRow;
import it.reyboz.bustorino.backend.Fetcher; import it.reyboz.bustorino.backend.Fetcher;
import it.reyboz.bustorino.backend.networkTools; import it.reyboz.bustorino.backend.networkTools;
import it.reyboz.bustorino.data.gtfs.CsvTableInserter; import it.reyboz.bustorino.data.gtfs.CsvTableInserter;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Attributes; import org.jsoup.nodes.Attributes;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.jsoup.select.Elements; import org.jsoup.select.Elements;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection; import java.net.HttpURLConnection;
import java.net.URL; import java.net.URL;
import java.text.ParseException; import java.text.ParseException;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.*; import java.util.*;
import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.zip.ZipEntry; import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipInputStream; import java.util.zip.ZipInputStream;
abstract public class GtfsDataParser { abstract public class GtfsDataParser {
public static final String GTFS_ADDRESS="https://www.gtt.to.it/open_data/gtt_gtfs.zip"; public static final String GTFS_ADDRESS="https://www.gtt.to.it/open_data/gtt_gtfs.zip";
public static final String GTFS_PAGE_ADDRESS="http://aperto.comune.torino.it/dataset/feed-gtfs-trasporti-gtt"; public static final String GTFS_PAGE_ADDRESS="http://aperto.comune.torino.it/dataset/feed-gtfs-trasporti-gtt";
private static final String DEBUG_TAG = "BusTO-GTFSDataParser"; private static final String DEBUG_TAG = "BusTO-GTFSDataParser";
▲ Show 20 Lines • Show All 103 Lines • ▼ Show 20 Lines public static Date getLastGTFSUpdateDate(AtomicReference<Fetcher.Result> res) {
} }
break; break;
} }
} }
res.set(Fetcher.Result.PARSER_ERROR); res.set(Fetcher.Result.PARSER_ERROR);
return finalDate; return finalDate;
} }
public static void readGtfsZipEntry(ZipEntry entry, ZipFile zipFile, Context con) throws IOException{
String tableName = entry.getName().split("\\.")[0].trim();
InputStream stream = zipFile.getInputStream(entry);
String s = String.format(Locale.ENGLISH, "Entry: %s len %d added",
entry.getName(),
entry.getSize()
);
Log.d(DEBUG_TAG, s);
final BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
GtfsDataParser.readCSVWithColumns(reader, tableName, con);
stream.close();
}
public static void readCSVWithColumns(BufferedReader reader, String tableName, Context con) throws IOException { public static void readCSVWithColumns(BufferedReader reader, String tableName, Context con) throws IOException {
//String[] elements; //String[] elements;
List<String> lineElements; List<String> lineElements;
String line; String line;
final String header = reader.readLine(); /*final String header = reader.readLine();
if (header == null){ if (header == null){
throw new IOException(); throw new IOException();
} }*/
//elements = header.split("\n")[0].split(","); //elements = header.split("\n")[0].split(",");
//System.out.println(Arrays.toString(elements)); //System.out.println(Arrays.toString(elements));
lineElements = readCsvLine(header); //lineElements = readCsvLine(header);
NamedCsvReader csvReader = NamedCsvReader.builder().build(reader);
CloseableIterator<NamedCsvRow> iterator = csvReader.iterator();
final HashMap<Integer,String> columnMap = new HashMap<>();
final CsvTableInserter inserter = new CsvTableInserter(tableName,con); final CsvTableInserter inserter = new CsvTableInserter(tableName,con);
/*final HashMap<Integer,String> columnMap = new HashMap<>();
for (int i=0; i< lineElements.size(); i++){ for (int i=0; i< lineElements.size(); i++){
//columnMap.put(i, fixStringIfItHasQuotes(elements[i].trim()) ); //columnMap.put(i, fixStringIfItHasQuotes(elements[i].trim()) );
columnMap.put(i, lineElements.get(i).trim() ); columnMap.put(i, lineElements.get(i).trim() );
} }
Log.d(DEBUG_TAG, "Columns for the file: "+columnMap); Log.d(DEBUG_TAG, "Columns for the file: "+columnMap);
boolean first = true; boolean first = true;
while((line = reader.readLine())!=null){ while((line = reader.readLine())!=null){
//there is a line of data //there is a line of data
//elements = line.split("\n")[0].split(","); //elements = line.split("\n")[0].split(",");
if(first) Log.d(DEBUG_TAG, "Element line: "+line); if(first) Log.d(DEBUG_TAG, "Element line: "+line);
lineElements = readCsvLine(line); lineElements = readCsvLine(line);
final Map<String,String> rowsMap = getColumnsAsString(lineElements.toArray(new String[0]), columnMap); final Map<String,String> rowsMap = getColumnsAsString(lineElements.toArray(new String[0]), columnMap);
if (first){ if (first){
Log.d(DEBUG_TAG, " in map:"+rowsMap); Log.d(DEBUG_TAG, " in map:"+rowsMap);
first=false; first=false;
} }
inserter.addElement(rowsMap); inserter.addElement(rowsMap);
}*/
int c = 0;
while (iterator.hasNext()){
final Map<String,String> rowsMap = iterator.next().getFields();
if (c < 1){
Log.d(DEBUG_TAG, " in map:"+rowsMap);
c++;
}
inserter.addElement(rowsMap);
} }
//commit data //commit data
inserter.insertDataInDatabase(); inserter.finishInsert();
} }
@NonNull @NonNull
private static Map<String,String> getColumnsAsString(@NonNull String[] lineElements, Map<Integer,String> colsIndices) private static Map<String,String> getColumnsAsString(@NonNull String[] lineElements, Map<Integer,String> colsIndices)
{ {
final HashMap<String,String> theMap = new HashMap<>(); final HashMap<String,String> theMap = new HashMap<>();
for(int l=0; l<lineElements.length; l++){ for(int l=0; l<lineElements.length; l++){
if(!colsIndices.containsKey(l)) if(!colsIndices.containsKey(l))
continue; continue;
▲ Show 20 Lines • Show All 57 Lines • Show Last 20 Lines
View Options

includes/class-QueryRoom.php

package it.reyboz.bustorino.backend.gtfs; package it.reyboz.bustorino.backend.gtfs;
import android.content.Context; import android.content.Context;
import android.util.Log; import android.util.Log;
import androidx.annotation.NonNull; import androidx.annotation.NonNull;
import de.siegmar.fastcsv.reader.CloseableIterator;
import de.siegmar.fastcsv.reader.NamedCsvReader;
import de.siegmar.fastcsv.reader.NamedCsvRow;
import it.reyboz.bustorino.backend.Fetcher; import it.reyboz.bustorino.backend.Fetcher;
import it.reyboz.bustorino.backend.networkTools; import it.reyboz.bustorino.backend.networkTools;
import it.reyboz.bustorino.data.gtfs.CsvTableInserter; import it.reyboz.bustorino.data.gtfs.CsvTableInserter;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Attributes; import org.jsoup.nodes.Attributes;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.jsoup.select.Elements; import org.jsoup.select.Elements;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection; import java.net.HttpURLConnection;
import java.net.URL; import java.net.URL;
import java.text.ParseException; import java.text.ParseException;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.*; import java.util.*;
import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.zip.ZipEntry; import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipInputStream; import java.util.zip.ZipInputStream;
abstract public class GtfsDataParser { abstract public class GtfsDataParser {
public static final String GTFS_ADDRESS="https://www.gtt.to.it/open_data/gtt_gtfs.zip"; public static final String GTFS_ADDRESS="https://www.gtt.to.it/open_data/gtt_gtfs.zip";
public static final String GTFS_PAGE_ADDRESS="http://aperto.comune.torino.it/dataset/feed-gtfs-trasporti-gtt"; public static final String GTFS_PAGE_ADDRESS="http://aperto.comune.torino.it/dataset/feed-gtfs-trasporti-gtt";
private static final String DEBUG_TAG = "BusTO-GTFSDataParser"; private static final String DEBUG_TAG = "BusTO-GTFSDataParser";
▲ Show 20 Lines • Show All 103 Lines • ▼ Show 20 Lines public static Date getLastGTFSUpdateDate(AtomicReference<Fetcher.Result> res) {
} }
break; break;
} }
} }
res.set(Fetcher.Result.PARSER_ERROR); res.set(Fetcher.Result.PARSER_ERROR);
return finalDate; return finalDate;
} }
public static void readGtfsZipEntry(ZipEntry entry, ZipFile zipFile, Context con) throws IOException{
String tableName = entry.getName().split("\\.")[0].trim();
InputStream stream = zipFile.getInputStream(entry);
String s = String.format(Locale.ENGLISH, "Entry: %s len %d added",
entry.getName(),
entry.getSize()
);
Log.d(DEBUG_TAG, s);
final BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
GtfsDataParser.readCSVWithColumns(reader, tableName, con);
stream.close();
}
public static void readCSVWithColumns(BufferedReader reader, String tableName, Context con) throws IOException { public static void readCSVWithColumns(BufferedReader reader, String tableName, Context con) throws IOException {
//String[] elements; //String[] elements;
List<String> lineElements; List<String> lineElements;
String line; String line;
final String header = reader.readLine(); /*final String header = reader.readLine();
if (header == null){ if (header == null){
throw new IOException(); throw new IOException();
} }*/
//elements = header.split("\n")[0].split(","); //elements = header.split("\n")[0].split(",");
//System.out.println(Arrays.toString(elements)); //System.out.println(Arrays.toString(elements));
lineElements = readCsvLine(header); //lineElements = readCsvLine(header);
NamedCsvReader csvReader = NamedCsvReader.builder().build(reader);
CloseableIterator<NamedCsvRow> iterator = csvReader.iterator();
final HashMap<Integer,String> columnMap = new HashMap<>();
final CsvTableInserter inserter = new CsvTableInserter(tableName,con); final CsvTableInserter inserter = new CsvTableInserter(tableName,con);
/*final HashMap<Integer,String> columnMap = new HashMap<>();
for (int i=0; i< lineElements.size(); i++){ for (int i=0; i< lineElements.size(); i++){
//columnMap.put(i, fixStringIfItHasQuotes(elements[i].trim()) ); //columnMap.put(i, fixStringIfItHasQuotes(elements[i].trim()) );
columnMap.put(i, lineElements.get(i).trim() ); columnMap.put(i, lineElements.get(i).trim() );
} }
Log.d(DEBUG_TAG, "Columns for the file: "+columnMap); Log.d(DEBUG_TAG, "Columns for the file: "+columnMap);
boolean first = true; boolean first = true;
while((line = reader.readLine())!=null){ while((line = reader.readLine())!=null){
//there is a line of data //there is a line of data
//elements = line.split("\n")[0].split(","); //elements = line.split("\n")[0].split(",");
if(first) Log.d(DEBUG_TAG, "Element line: "+line); if(first) Log.d(DEBUG_TAG, "Element line: "+line);
lineElements = readCsvLine(line); lineElements = readCsvLine(line);
final Map<String,String> rowsMap = getColumnsAsString(lineElements.toArray(new String[0]), columnMap); final Map<String,String> rowsMap = getColumnsAsString(lineElements.toArray(new String[0]), columnMap);
if (first){ if (first){
Log.d(DEBUG_TAG, " in map:"+rowsMap); Log.d(DEBUG_TAG, " in map:"+rowsMap);
first=false; first=false;
} }
inserter.addElement(rowsMap); inserter.addElement(rowsMap);
}*/
int c = 0;
while (iterator.hasNext()){
final Map<String,String> rowsMap = iterator.next().getFields();
if (c < 1){
Log.d(DEBUG_TAG, " in map:"+rowsMap);
c++;
}
inserter.addElement(rowsMap);
} }
//commit data //commit data
inserter.insertDataInDatabase(); inserter.finishInsert();
} }
@NonNull @NonNull
private static Map<String,String> getColumnsAsString(@NonNull String[] lineElements, Map<Integer,String> colsIndices) private static Map<String,String> getColumnsAsString(@NonNull String[] lineElements, Map<Integer,String> colsIndices)
{ {
final HashMap<String,String> theMap = new HashMap<>(); final HashMap<String,String> theMap = new HashMap<>();
for(int l=0; l<lineElements.length; l++){ for(int l=0; l<lineElements.length; l++){
if(!colsIndices.containsKey(l)) if(!colsIndices.containsKey(l))
continue; continue;
▲ Show 20 Lines • Show All 57 Lines • Show Last 20 Lines
View Options

includes/class-QueryTrack.php

Loading...
Public contents are in Creative Commons Attribution-ShareAlike 4.0 (CC-BY-SA) or GNU Free Documentation License (at your option) unless otherwise noted. · Contact / Register