Changeset View
Changeset View
Standalone View
Standalone View
src/it/reyboz/bustorino/backend/gtfs/GtfsDataParser.java
package it.reyboz.bustorino.backend.gtfs; | package it.reyboz.bustorino.backend.gtfs; | ||||
import android.content.Context; | import android.content.Context; | ||||
import android.util.Log; | import android.util.Log; | ||||
import androidx.annotation.NonNull; | import androidx.annotation.NonNull; | ||||
import de.siegmar.fastcsv.reader.CloseableIterator; | |||||
import de.siegmar.fastcsv.reader.NamedCsvReader; | |||||
import de.siegmar.fastcsv.reader.NamedCsvRow; | |||||
import it.reyboz.bustorino.backend.Fetcher; | import it.reyboz.bustorino.backend.Fetcher; | ||||
import it.reyboz.bustorino.backend.networkTools; | import it.reyboz.bustorino.backend.networkTools; | ||||
import it.reyboz.bustorino.data.gtfs.CsvTableInserter; | import it.reyboz.bustorino.data.gtfs.CsvTableInserter; | ||||
import org.jsoup.Jsoup; | import org.jsoup.Jsoup; | ||||
import org.jsoup.nodes.Attributes; | import org.jsoup.nodes.Attributes; | ||||
import org.jsoup.nodes.Document; | import org.jsoup.nodes.Document; | ||||
import org.jsoup.nodes.Element; | import org.jsoup.nodes.Element; | ||||
import org.jsoup.select.Elements; | import org.jsoup.select.Elements; | ||||
import java.io.BufferedReader; | import java.io.BufferedReader; | ||||
import java.io.IOException; | import java.io.IOException; | ||||
import java.io.InputStream; | import java.io.InputStream; | ||||
import java.io.InputStreamReader; | |||||
import java.net.HttpURLConnection; | import java.net.HttpURLConnection; | ||||
import java.net.URL; | import java.net.URL; | ||||
import java.text.ParseException; | import java.text.ParseException; | ||||
import java.text.SimpleDateFormat; | import java.text.SimpleDateFormat; | ||||
import java.util.*; | import java.util.*; | ||||
import java.util.concurrent.atomic.AtomicReference; | import java.util.concurrent.atomic.AtomicReference; | ||||
import java.util.regex.Matcher; | import java.util.regex.Matcher; | ||||
import java.util.regex.Pattern; | import java.util.regex.Pattern; | ||||
import java.util.zip.ZipEntry; | import java.util.zip.ZipEntry; | ||||
import java.util.zip.ZipFile; | |||||
import java.util.zip.ZipInputStream; | import java.util.zip.ZipInputStream; | ||||
abstract public class GtfsDataParser { | abstract public class GtfsDataParser { | ||||
public static final String GTFS_ADDRESS="https://www.gtt.to.it/open_data/gtt_gtfs.zip"; | public static final String GTFS_ADDRESS="https://www.gtt.to.it/open_data/gtt_gtfs.zip"; | ||||
public static final String GTFS_PAGE_ADDRESS="http://aperto.comune.torino.it/dataset/feed-gtfs-trasporti-gtt"; | public static final String GTFS_PAGE_ADDRESS="http://aperto.comune.torino.it/dataset/feed-gtfs-trasporti-gtt"; | ||||
private static final String DEBUG_TAG = "BusTO-GTFSDataParser"; | private static final String DEBUG_TAG = "BusTO-GTFSDataParser"; | ||||
▲ Show 20 Lines • Show All 103 Lines • ▼ Show 20 Lines | public static Date getLastGTFSUpdateDate(AtomicReference<Fetcher.Result> res) { | ||||
} | } | ||||
break; | break; | ||||
} | } | ||||
} | } | ||||
res.set(Fetcher.Result.PARSER_ERROR); | res.set(Fetcher.Result.PARSER_ERROR); | ||||
return finalDate; | return finalDate; | ||||
} | } | ||||
public static void readGtfsZipEntry(ZipEntry entry, ZipFile zipFile, Context con) throws IOException{ | |||||
String tableName = entry.getName().split("\\.")[0].trim(); | |||||
InputStream stream = zipFile.getInputStream(entry); | |||||
String s = String.format(Locale.ENGLISH, "Entry: %s len %d added", | |||||
entry.getName(), | |||||
entry.getSize() | |||||
); | |||||
Log.d(DEBUG_TAG, s); | |||||
final BufferedReader reader = new BufferedReader(new InputStreamReader(stream)); | |||||
GtfsDataParser.readCSVWithColumns(reader, tableName, con); | |||||
stream.close(); | |||||
} | |||||
public static void readCSVWithColumns(BufferedReader reader, String tableName, Context con) throws IOException { | public static void readCSVWithColumns(BufferedReader reader, String tableName, Context con) throws IOException { | ||||
//String[] elements; | //String[] elements; | ||||
List<String> lineElements; | List<String> lineElements; | ||||
String line; | String line; | ||||
final String header = reader.readLine(); | /*final String header = reader.readLine(); | ||||
if (header == null){ | if (header == null){ | ||||
throw new IOException(); | throw new IOException(); | ||||
} | }*/ | ||||
//elements = header.split("\n")[0].split(","); | //elements = header.split("\n")[0].split(","); | ||||
//System.out.println(Arrays.toString(elements)); | //System.out.println(Arrays.toString(elements)); | ||||
lineElements = readCsvLine(header); | //lineElements = readCsvLine(header); | ||||
NamedCsvReader csvReader = NamedCsvReader.builder().build(reader); | |||||
CloseableIterator<NamedCsvRow> iterator = csvReader.iterator(); | |||||
final HashMap<Integer,String> columnMap = new HashMap<>(); | |||||
final CsvTableInserter inserter = new CsvTableInserter(tableName,con); | final CsvTableInserter inserter = new CsvTableInserter(tableName,con); | ||||
/*final HashMap<Integer,String> columnMap = new HashMap<>(); | |||||
for (int i=0; i< lineElements.size(); i++){ | for (int i=0; i< lineElements.size(); i++){ | ||||
//columnMap.put(i, fixStringIfItHasQuotes(elements[i].trim()) ); | //columnMap.put(i, fixStringIfItHasQuotes(elements[i].trim()) ); | ||||
columnMap.put(i, lineElements.get(i).trim() ); | columnMap.put(i, lineElements.get(i).trim() ); | ||||
} | } | ||||
Log.d(DEBUG_TAG, "Columns for the file: "+columnMap); | Log.d(DEBUG_TAG, "Columns for the file: "+columnMap); | ||||
boolean first = true; | boolean first = true; | ||||
while((line = reader.readLine())!=null){ | while((line = reader.readLine())!=null){ | ||||
//there is a line of data | //there is a line of data | ||||
//elements = line.split("\n")[0].split(","); | //elements = line.split("\n")[0].split(","); | ||||
if(first) Log.d(DEBUG_TAG, "Element line: "+line); | if(first) Log.d(DEBUG_TAG, "Element line: "+line); | ||||
lineElements = readCsvLine(line); | lineElements = readCsvLine(line); | ||||
final Map<String,String> rowsMap = getColumnsAsString(lineElements.toArray(new String[0]), columnMap); | final Map<String,String> rowsMap = getColumnsAsString(lineElements.toArray(new String[0]), columnMap); | ||||
if (first){ | if (first){ | ||||
Log.d(DEBUG_TAG, " in map:"+rowsMap); | Log.d(DEBUG_TAG, " in map:"+rowsMap); | ||||
first=false; | first=false; | ||||
} | } | ||||
inserter.addElement(rowsMap); | inserter.addElement(rowsMap); | ||||
}*/ | |||||
int c = 0; | |||||
while (iterator.hasNext()){ | |||||
final Map<String,String> rowsMap = iterator.next().getFields(); | |||||
if (c < 1){ | |||||
Log.d(DEBUG_TAG, " in map:"+rowsMap); | |||||
c++; | |||||
} | |||||
inserter.addElement(rowsMap); | |||||
} | } | ||||
//commit data | //commit data | ||||
inserter.insertDataInDatabase(); | inserter.finishInsert(); | ||||
} | } | ||||
@NonNull | @NonNull | ||||
private static Map<String,String> getColumnsAsString(@NonNull String[] lineElements, Map<Integer,String> colsIndices) | private static Map<String,String> getColumnsAsString(@NonNull String[] lineElements, Map<Integer,String> colsIndices) | ||||
{ | { | ||||
final HashMap<String,String> theMap = new HashMap<>(); | final HashMap<String,String> theMap = new HashMap<>(); | ||||
for(int l=0; l<lineElements.length; l++){ | for(int l=0; l<lineElements.length; l++){ | ||||
if(!colsIndices.containsKey(l)) | if(!colsIndices.containsKey(l)) | ||||
continue; | continue; | ||||
▲ Show 20 Lines • Show All 57 Lines • Show Last 20 Lines |
Public contents are in Creative Commons Attribution-ShareAlike 4.0 (CC-BY-SA) or GNU Free Documentation License (at your option) unless otherwise noted. · Contact / Register