diff --git a/build.gradle b/build.gradle
index abc6a77..41968bd 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1,140 +1,144 @@
buildscript {
repositories {
jcenter()
maven { url 'https://maven.google.com' }
google()
}
ext {
//multidex
multidex_version = "2.0.1"
//libraries versions
fragment_version = "1.3.6"
activity_version = "1.2.4"
appcompat_version = "1.3.1"
preference_version = "1.1.1"
work_version = "2.5.0"
acra_version = "5.7.0"
lifecycle_version = "2.3.1"
arch_version = "2.1.0"
room_version = "2.3.0"
//kotlin
kotlin_version = '1.5.0'
coroutines_version = "1.5.0"
}
dependencies {
classpath 'com.android.tools.build:gradle:4.1.3'
classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
}
}
allprojects {
repositories {
jcenter()
maven { url 'https://maven.google.com' }
google()
+ mavenCentral()
}
}
apply plugin: 'com.android.application'
apply plugin: 'kotlin-android'
apply plugin: 'kotlin-kapt'
android {
compileSdkVersion 29
buildToolsVersion '29.0.3'
defaultConfig {
applicationId "it.reyboz.bustorino"
minSdkVersion 15
targetSdkVersion 29
versionCode 35
versionName "1.15.4"
vectorDrawables.useSupportLibrary = true
multiDexEnabled true
}
compileOptions {
sourceCompatibility JavaVersion.VERSION_1_8
targetCompatibility JavaVersion.VERSION_1_8
}
sourceSets {
main {
manifest.srcFile 'AndroidManifest.xml'
java.srcDirs = ['src']
resources.srcDirs = ['src']
aidl.srcDirs = ['src']
renderscript.srcDirs = ['src']
res.srcDirs = ['res']
assets.srcDirs = ['assets']
}
}
buildTypes {
debug {
applicationIdSuffix ".debug"
versionNameSuffix "-dev"
}
}
lintOptions {
abortOnError false
}
repositories {
jcenter()
mavenLocal()
}
dependencies {
//new libraries
implementation "androidx.fragment:fragment:$fragment_version"
implementation "androidx.activity:activity:$activity_version"
implementation "androidx.annotation:annotation:1.2.0"
implementation "androidx.swiperefreshlayout:swiperefreshlayout:1.1.0"
implementation "androidx.appcompat:appcompat:$appcompat_version"
implementation "androidx.appcompat:appcompat-resources:$appcompat_version"
implementation "androidx.preference:preference:$preference_version"
implementation "androidx.work:work-runtime:$work_version"
implementation "com.google.android.material:material:1.4.0"
implementation 'androidx.constraintlayout:constraintlayout:2.0.4'
implementation 'org.jsoup:jsoup:1.13.1'
implementation 'com.readystatesoftware.sqliteasset:sqliteassethelper:2.0.1'
implementation 'com.android.volley:volley:1.2.0'
implementation 'org.osmdroid:osmdroid-android:6.1.10'
// ACRA
implementation "ch.acra:acra-mail:$acra_version"
implementation "ch.acra:acra-dialog:$acra_version"
// google transit realtime
implementation 'com.google.protobuf:protobuf-java:3.14.0'
// ViewModel
implementation "androidx.lifecycle:lifecycle-viewmodel:$lifecycle_version"
// LiveData
implementation "androidx.lifecycle:lifecycle-livedata:$lifecycle_version"
// Lifecycles only (without ViewModel or LiveData)
implementation "androidx.lifecycle:lifecycle-runtime:$lifecycle_version"
// Room components
implementation "androidx.room:room-ktx:$room_version"
kapt "androidx.room:room-compiler:$room_version"
androidTestImplementation "androidx.room:room-testing:$room_version"
//multidex - we need this to build the app
implementation "androidx.multidex:multidex:$multidex_version"
+
+ implementation 'de.siegmar:fastcsv:2.0.0'
+
}
}
dependencies {
implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
api "org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutines_version"
api "org.jetbrains.kotlinx:kotlinx-coroutines-android:$coroutines_version"
}
\ No newline at end of file
diff --git a/res/layout/activity_experiments.xml b/res/layout/activity_experiments.xml
index 545a44c..e2fa740 100644
--- a/res/layout/activity_experiments.xml
+++ b/res/layout/activity_experiments.xml
@@ -1,27 +1,40 @@
+ android:id="@+id/button"
+ android:layout_width="wrap_content"
+ android:layout_height="wrap_content"
+ android:onClick="runExp"
+ android:text="Download GTFS data"
+ app:layout_constraintBottom_toBottomOf="parent"
+ app:layout_constraintEnd_toEndOf="parent"
+ app:layout_constraintHorizontal_bias="0.497"
+ app:layout_constraintStart_toStartOf="parent"
+ app:layout_constraintTop_toTopOf="parent"
+ app:layout_constraintVertical_bias="0.292" />
+
+
\ No newline at end of file
diff --git a/src/it/reyboz/bustorino/ActivityExperiments.java b/src/it/reyboz/bustorino/ActivityExperiments.java
index 825ac88..35624da 100644
--- a/src/it/reyboz/bustorino/ActivityExperiments.java
+++ b/src/it/reyboz/bustorino/ActivityExperiments.java
@@ -1,141 +1,196 @@
/*
BusTO - Data components
Copyright (C) 2021 Fabio Mazza
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see .
*/
package it.reyboz.bustorino;
import android.content.Context;
import android.os.AsyncTask;
import android.os.Handler;
import android.os.Looper;
import android.util.Log;
import android.view.View;
import android.widget.Button;
import android.widget.Toast;
import androidx.appcompat.app.AppCompatActivity;
import android.os.Bundle;
import it.reyboz.bustorino.backend.Fetcher;
import it.reyboz.bustorino.backend.gtfs.GtfsDataParser;
import it.reyboz.bustorino.backend.networkTools;
import it.reyboz.bustorino.backend.utils;
+import it.reyboz.bustorino.data.gtfs.GtfsDatabase;
+import it.reyboz.bustorino.data.gtfs.StaticGtfsDao;
import it.reyboz.bustorino.middleware.GeneralActivity;
import java.io.*;
import java.net.MalformedURLException;
import java.net.URL;
-import java.util.Date;
-import java.util.List;
-import java.util.Locale;
+import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicReference;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipInputStream;
public class ActivityExperiments extends GeneralActivity {
+ ExecutorService executorService;
+ final static String DEBUG_TAG = "ExperimentsGTFS";
+
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_experiments);
Button deleteButton = findViewById(R.id.deleteButton);
if(deleteButton!=null)
deleteButton.setOnClickListener(view -> {
File saveFile = new File(getFilesDir(), "gtfs_data.zip");
if(!saveFile.isDirectory() && saveFile.exists()){
//delete the file
if(saveFile.delete())
Toast.makeText(this, "Gtfs zip deleted", Toast.LENGTH_SHORT).show();
else
Toast.makeText(this, "Cannot delete gtfs zip", Toast.LENGTH_SHORT).show();
} else
Toast.makeText(this, "Gtfs data zip not present", Toast.LENGTH_SHORT).show();
});
+
+ Button cleanDBButton = findViewById(R.id.deleteDbButton);
+ if(cleanDBButton!=null)
+ cleanDBButton.setOnClickListener(this::deleteDatabase);
+
+ executorService = Executors.newFixedThreadPool(2);
}
public void runExp(View v){
final Context appContext = v.getContext().getApplicationContext();
Runnable run = new Runnable() {
@Override
public void run() {
- final String DEBUG_TAG = "ExperimentsGTFS";
+
AtomicReference res = new AtomicReference<>();
//List files = GtfsDataParser.readFilesList(res);
Date updateDate = GtfsDataParser.getLastGTFSUpdateDate(res);
Log.w(
"ExperimentGTFS", "Last update date is " + updateDate//utils.joinList(files, "\n")
);
//Toast.makeText(v.getContext(), "Gtfs data already downloaded", Toast.LENGTH_SHORT).show();
+ StaticGtfsDao dao = GtfsDatabase.Companion.getGtfsDatabase(appContext).gtfsDao();
+ Log.d(DEBUG_TAG, String.valueOf(dao));
+ dao.deleteAllStopTimes();
+
File saveFile = new File(getFilesDir(), "gtfs_data.zip");
if (!saveFile.isDirectory() && saveFile.exists()) {
Log.w(DEBUG_TAG, "Zip exists: " + saveFile);
- try (FileInputStream fileStream = new FileInputStream(saveFile)) {
- ZipInputStream stream = new ZipInputStream(fileStream);
+
+
+ try (ZipFile zipFile = new ZipFile(saveFile)) {
+ //ZipInputStream stream = new ZipInputStream(fileStream);
// now iterate through each item in the stream. The get next
// entry call will return a ZipEntry for each file in the
// stream
+ Enumeration extends ZipEntry> entries = zipFile.entries();
ZipEntry entry;
String line;
- final BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
+ //final BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
+ HashSet readLater = new HashSet<>();
+ while(entries.hasMoreElements()){
+ entry = entries.nextElement();
+ //String tableName = entry.getName().split("\\.")[0].trim();
+ if(entry.getName().trim().equals("stop_times.txt")) {
+ readLater.add(entry);
+ continue;
+ }
+ GtfsDataParser.readGtfsZipEntry(entry, zipFile, v.getContext().getApplicationContext());
+ }
+ for(ZipEntry laterEntry: readLater){
+ GtfsDataParser.readGtfsZipEntry(laterEntry, zipFile, v.getContext().getApplicationContext());
+ }
+ //Toast.makeText(appContext, "D", Toast.LENGTH_SHORT).show();
+ /*
while ((entry = stream.getNextEntry()) != null) {
String s = String.format(Locale.ENGLISH, "Entry: %s len %d added",
entry.getName(),
entry.getSize()
);
+ if(entry.getName().contains("stop_times.")){
+ //skip and do later
+
+ }
//Toast.makeText(v.getContext(), "File: " + entry.getName(), Toast.LENGTH_SHORT).show();
Log.d(DEBUG_TAG, s);
//read data in table
final String tableName = entry.getName().split("\\.")[0].trim();
- GtfsDataParser.readCSVWithColumns(reader, tableName, v.getContext().getApplicationContext());
+
// Once we get the entry from the stream, the stream is
// positioned read to read the raw data, and we keep
// reading until read returns 0 or less.
//result.add(entry.getName());
}
stream.close();
+ */
} catch (IOException e) {
e.printStackTrace();
}
//saveFile.delete();
} else
try {
//Toast.makeText(v.getContext(), "Downloading gtfs data", Toast.LENGTH_SHORT).show();
networkTools.saveFileInCache(saveFile, new URL(GtfsDataParser.GTFS_ADDRESS));
Log.w(DEBUG_TAG, "File saved");
} catch (MalformedURLException e) {
e.printStackTrace();
}
}
};
- ExecutorService executorService = Executors.newFixedThreadPool(2);
+
+ Toast.makeText(this, "Launching, no result will show", Toast.LENGTH_SHORT).show();
//Looper looper = new Looper(true);
//Handler handler = new Handler();
//handler.post(run);
executorService.execute(run);
}
+ public void deleteDatabase(View v){
+ final Context con = getApplicationContext().getApplicationContext();
+ Toast.makeText(this, "Deleting GTFS DB contents, wait a few seconds", Toast.LENGTH_SHORT).show();
+ Runnable deleteDB = new Runnable() {
+ @Override
+ public void run() {
+ StaticGtfsDao dao = GtfsDatabase.Companion.getGtfsDatabase(con).gtfsDao();
+ Log.d(DEBUG_TAG, String.valueOf(dao));
+ dao.deleteAllStopTimes();
+ dao.deleteAllTrips();
+ dao.deleteAllRoutes();
+ dao.deleteAllStops();
+ dao.deleteAllServices();
+ Log.d(DEBUG_TAG, "Deleted stuff");
+ }
+ };
+ executorService.execute(deleteDB);
+ }
}
\ No newline at end of file
diff --git a/src/it/reyboz/bustorino/backend/gtfs/GtfsDataParser.java b/src/it/reyboz/bustorino/backend/gtfs/GtfsDataParser.java
index e2a4b40..419642f 100644
--- a/src/it/reyboz/bustorino/backend/gtfs/GtfsDataParser.java
+++ b/src/it/reyboz/bustorino/backend/gtfs/GtfsDataParser.java
@@ -1,256 +1,285 @@
package it.reyboz.bustorino.backend.gtfs;
import android.content.Context;
import android.util.Log;
import androidx.annotation.NonNull;
+
+import de.siegmar.fastcsv.reader.CloseableIterator;
+import de.siegmar.fastcsv.reader.NamedCsvReader;
+import de.siegmar.fastcsv.reader.NamedCsvRow;
import it.reyboz.bustorino.backend.Fetcher;
import it.reyboz.bustorino.backend.networkTools;
import it.reyboz.bustorino.data.gtfs.CsvTableInserter;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Attributes;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
+import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
import java.util.zip.ZipInputStream;
abstract public class GtfsDataParser {
public static final String GTFS_ADDRESS="https://www.gtt.to.it/open_data/gtt_gtfs.zip";
public static final String GTFS_PAGE_ADDRESS="http://aperto.comune.torino.it/dataset/feed-gtfs-trasporti-gtt";
private static final String DEBUG_TAG = "BusTO-GTFSDataParser";
private static final Pattern quotePattern = Pattern.compile("^\\s*\"((?:[^\"]|(?:\"\"))*?)\"\\s*,");
/**
* First trial for a function to download the zip
* @param res Fetcher.result
* @return the list of files inside the ziè
*/
public static ArrayList readFilesList(AtomicReference res){
HttpURLConnection urlConnection;
InputStream in;
ArrayList result = new ArrayList<>();
try {
final URL gtfsUrl = new URL(GTFS_ADDRESS);
urlConnection = (HttpURLConnection) gtfsUrl.openConnection();
} catch(IOException e) {
//e.printStackTrace();
res.set(Fetcher.Result.SERVER_ERROR); // even when offline, urlConnection works fine. WHY.
return null;
}
urlConnection.setConnectTimeout(4000);
urlConnection.setReadTimeout(50*1000);
try {
in = urlConnection.getInputStream();
} catch (Exception e) {
try {
if(urlConnection.getResponseCode()==404)
res.set(Fetcher.Result.SERVER_ERROR_404);
} catch (IOException e2) {
e2.printStackTrace();
}
return null;
}
try (ZipInputStream stream = new ZipInputStream(in)) {
// now iterate through each item in the stream. The get next
// entry call will return a ZipEntry for each file in the
// stream
ZipEntry entry;
while ((entry = stream.getNextEntry()) != null) {
String s = String.format(Locale.ENGLISH, "Entry: %s len %d added",
entry.getName(),
entry.getSize()
);
System.out.println(s);
// Once we get the entry from the stream, the stream is
// positioned read to read the raw data, and we keep
// reading until read returns 0 or less.
result.add(entry.getName());
}
} catch (IOException e) {
e.printStackTrace();
}
// we must always close the zip file.
return result;
}
public static Date getLastGTFSUpdateDate(AtomicReference res) {
URL theURL;
final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ", Locale.ENGLISH);
//final Date baseDate = dateFormat.parse("1970-00-00T00:00:00+0000");
final Date nullDate = new Date(0);
try{
theURL = new URL(GTFS_PAGE_ADDRESS);
} catch (IOException ex){
Log.e(DEBUG_TAG, "Fixed URL is null, this is a real issue");
return nullDate;
}
res.set(Fetcher.Result.OK);
final String fullPageDOM = networkTools.getDOM(theURL, res);
if(fullPageDOM== null){
//Something wrong happend
Log.e(DEBUG_TAG, "Cannot get URL");
return nullDate;
}
res.set(Fetcher.Result.OK);
Document doc = Jsoup.parse(fullPageDOM);
Elements sections = doc.select("section.additional-info");
Date finalDate = new Date(0);
for (Element sec: sections){
Element head = sec.select("h3").first();
String headTitle = head.text();
if(!headTitle.trim().toLowerCase(Locale.ITALIAN).equals("informazioni supplementari"))
continue;
for (Element row: sec.select("tr")){
if(!row.selectFirst("th").text().trim()
.toLowerCase(Locale.ITALIAN).equals("ultimo aggiornamento"))
continue;
Attributes spanAttributes = row.selectFirst("td > span").attributes();
String dateAsString = spanAttributes.get("data-datetime");
try {
finalDate = dateFormat.parse(dateAsString);
return finalDate;
}catch (ParseException ex){
Log.e(DEBUG_TAG, "Wrong date for the last update of GTFS Data: "+dateAsString);
res.set(Fetcher.Result.PARSER_ERROR);
ex.printStackTrace();
}
break;
}
}
res.set(Fetcher.Result.PARSER_ERROR);
return finalDate;
}
+ public static void readGtfsZipEntry(ZipEntry entry, ZipFile zipFile, Context con) throws IOException{
+ String tableName = entry.getName().split("\\.")[0].trim();
+ InputStream stream = zipFile.getInputStream(entry);
+ String s = String.format(Locale.ENGLISH, "Entry: %s len %d added",
+ entry.getName(),
+ entry.getSize()
+ );
+ Log.d(DEBUG_TAG, s);
+ final BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
+ GtfsDataParser.readCSVWithColumns(reader, tableName, con);
+ stream.close();
+ }
public static void readCSVWithColumns(BufferedReader reader, String tableName, Context con) throws IOException {
//String[] elements;
List lineElements;
String line;
- final String header = reader.readLine();
+ /*final String header = reader.readLine();
if (header == null){
throw new IOException();
- }
+ }*/
//elements = header.split("\n")[0].split(",");
//System.out.println(Arrays.toString(elements));
- lineElements = readCsvLine(header);
-
-
- final HashMap columnMap = new HashMap<>();
+ //lineElements = readCsvLine(header);
+ NamedCsvReader csvReader = NamedCsvReader.builder().build(reader);
+ CloseableIterator iterator = csvReader.iterator();
final CsvTableInserter inserter = new CsvTableInserter(tableName,con);
+ /*final HashMap columnMap = new HashMap<>();
+
for (int i=0; i< lineElements.size(); i++){
//columnMap.put(i, fixStringIfItHasQuotes(elements[i].trim()) );
columnMap.put(i, lineElements.get(i).trim() );
}
Log.d(DEBUG_TAG, "Columns for the file: "+columnMap);
boolean first = true;
while((line = reader.readLine())!=null){
//there is a line of data
//elements = line.split("\n")[0].split(",");
if(first) Log.d(DEBUG_TAG, "Element line: "+line);
lineElements = readCsvLine(line);
final Map rowsMap = getColumnsAsString(lineElements.toArray(new String[0]), columnMap);
if (first){
Log.d(DEBUG_TAG, " in map:"+rowsMap);
first=false;
}
inserter.addElement(rowsMap);
+ }*/
+ int c = 0;
+ while (iterator.hasNext()){
+
+ final Map rowsMap = iterator.next().getFields();
+ if (c < 1){
+ Log.d(DEBUG_TAG, " in map:"+rowsMap);
+ c++;
+ }
+ inserter.addElement(rowsMap);
}
//commit data
- inserter.insertDataInDatabase();
+ inserter.finishInsert();
}
@NonNull
private static Map getColumnsAsString(@NonNull String[] lineElements, Map colsIndices)
{
final HashMap theMap = new HashMap<>();
for(int l=0; l1){
//if(elements.length<3) throw new IllegalArgumentException("Malformed string");
return elements[1];
} else if(elements.length > 0)
return elements[0];
else
return item;
}
//https://stackoverflow.com/questions/7800494/parse-csv-with-double-quote-in-some-cases#7800519
public static List readCsvLine(String line) throws IllegalArgumentException
{
List list = new ArrayList();
line += ",";
for (int x = 0; x < line.length(); x++)
{
String s = line.substring(x);
if (s.trim().startsWith("\""))
{
Matcher m = quotePattern.matcher(s);
if (!m.find()) {
Log.e(DEBUG_TAG, "Cannot find pattern, "+s+" , line: "+line);
throw new IllegalArgumentException("CSV is malformed");
}
list.add(m.group(1).replace("\"\"", "\""));
x += m.end() - 1;
}
else
{
int y = s.indexOf(",");
if (y == -1)
throw new IllegalArgumentException("CSV is malformed");
list.add(s.substring(0, y));
x += y;
}
}
return list;
}
}
diff --git a/src/it/reyboz/bustorino/backend/networkTools.java b/src/it/reyboz/bustorino/backend/networkTools.java
index f0852af..808b629 100644
--- a/src/it/reyboz/bustorino/backend/networkTools.java
+++ b/src/it/reyboz/bustorino/backend/networkTools.java
@@ -1,224 +1,257 @@
/*
BusTO - Arrival times for Turin public transports.
Copyright (C) 2014 Valerio Bozzolan
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see .
*/
package it.reyboz.bustorino.backend;
import android.content.Context;
import androidx.annotation.Nullable;
import android.util.Log;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
+import java.util.Date;
import java.util.Map;
import java.util.Scanner;
import java.util.concurrent.atomic.AtomicReference;
public abstract class networkTools {
public static String getDOM(final URL url, final AtomicReference res) {
//Log.d("asyncwget", "Catching URL in background: " + uri[0]);
HttpURLConnection urlConnection;
StringBuilder result = null;
try {
urlConnection = (HttpURLConnection) url.openConnection();
} catch(IOException e) {
res.set(Fetcher.Result.SERVER_ERROR);
return null;
}
try {
InputStream in = new BufferedInputStream(
urlConnection.getInputStream());
BufferedReader reader = new BufferedReader(
new InputStreamReader(in));
result = new StringBuilder();
String line;
while ((line = reader.readLine()) != null) {
result.append(line);
}
} catch (Exception e) {
//Log.e("asyncwget", e.getMessage());
} finally {
if (urlConnection != null) {
urlConnection.disconnect();
}
}
if (result == null) {
res.set(Fetcher.Result.SERVER_ERROR);
return null;
}
res.set(Fetcher.Result.PARSER_ERROR); // will be set to "OK" later, this is a safety net in case StringBuilder returns null, the website returns an HTTP 204 or something like that.
return result.toString();
}
public static Fetcher.Result saveFileInCache(File outputFile, URL url) {
HttpURLConnection urlConnection;
try {
urlConnection = (HttpURLConnection) url.openConnection();
} catch (IOException e) {
//e.printStackTrace();
return Fetcher.Result.CONNECTION_ERROR;
}
urlConnection.setConnectTimeout(4000);
urlConnection.setReadTimeout(50 * 1000);
+ System.out.println("Last modified: "+new Date(urlConnection.getLastModified()));
Log.d("BusTO net Tools", "Download file "+url);
try (InputStream inputStream = urlConnection.getInputStream()) {
//File outputFile = new File(con.getFilesDir(), fileName);
//BufferedInputStream bufferedInputStream = new BufferedInputStream(inputStream);
FileOutputStream outputStream = new FileOutputStream(outputFile);
byte buffer[] = new byte[16384];
boolean inProgress = true;
while(inProgress){
int numread = inputStream.read(buffer);
inProgress = (numread > 0);
if(inProgress) outputStream.write(buffer, 0, numread);
}
outputStream.close();
//while (bufferedInputStream.available())
} catch (IOException e) {
e.printStackTrace();
try {
final Fetcher.Result res;
if(urlConnection.getResponseCode()==404)
res= Fetcher.Result.SERVER_ERROR_404;
else if(urlConnection.getResponseCode()!=200)
res= Fetcher.Result.SERVER_ERROR;
else res= Fetcher.Result.PARSER_ERROR;
urlConnection.disconnect();
return res;
} catch (IOException ioException) {
ioException.printStackTrace();
urlConnection.disconnect();
return Fetcher.Result.PARSER_ERROR;
}
}
urlConnection.disconnect();
return Fetcher.Result.OK;
}
+
+ @Nullable
+ public static Date checkLastModificationDate(URL url, AtomicReference res) {
+ HttpURLConnection urlConnection;
+ try {
+ urlConnection = (HttpURLConnection) url.openConnection();
+ } catch (IOException e) {
+ //e.printStackTrace();
+ res.set(Fetcher.Result.CONNECTION_ERROR);
+ return null;
+ }
+ urlConnection.setConnectTimeout(4000);
+ urlConnection.setReadTimeout(4 * 1000);
+ System.out.println("Last modified: "+new Date(urlConnection.getLastModified()));
+
+ Log.d("BusTO net Tools", "Download file "+url);
+ final Date theDate = new Date(urlConnection.getLastModified());
+
+ try {
+ if(urlConnection.getResponseCode()==404)
+ res.set(Fetcher.Result.SERVER_ERROR_404);
+ else if(urlConnection.getResponseCode()!=200)
+ res.set(Fetcher.Result.SERVER_ERROR);
+ } catch (IOException e) {
+ e.printStackTrace();
+ res.set(Fetcher.Result.PARSER_ERROR);
+ }
+ urlConnection.disconnect();
+ //theDate.getTime()
+ return theDate;
+ }
@Nullable
static String queryURL(URL url, AtomicReference res){
return queryURL(url,res,null);
}
@Nullable
static String queryURL(URL url, AtomicReference res, Map headers) {
HttpURLConnection urlConnection;
InputStream in;
String s;
try {
urlConnection = (HttpURLConnection) url.openConnection();
} catch(IOException e) {
//e.printStackTrace();
res.set(Fetcher.Result.SERVER_ERROR); // even when offline, urlConnection works fine. WHY.
return null;
}
// TODO: make this configurable?
urlConnection.setConnectTimeout(3000);
urlConnection.setReadTimeout(10000);
if(headers!= null){
for(String key : headers.keySet()){
urlConnection.setRequestProperty(key,headers.get(key));
}
}
res.set(Fetcher.Result.SERVER_ERROR); // will be set to OK later
try {
in = urlConnection.getInputStream();
} catch (Exception e) {
try {
if(urlConnection.getResponseCode()==404)
res.set(Fetcher.Result.SERVER_ERROR_404);
} catch (IOException e2) {
e2.printStackTrace();
}
return null;
}
//s = streamToString(in);
try {
final long startTime = System.currentTimeMillis();
s = parseStreamToString(in);
final long endtime = System.currentTimeMillis();
Log.d("NetworkTools-queryURL","reading response took "+(endtime-startTime)+" millisec");
} catch (IOException e) {
e.printStackTrace();
return null;
}
try {
in.close();
} catch(IOException ignored) {
//ignored.printStackTrace();
}
try {
urlConnection.disconnect();
} catch(Exception ignored) {
//ignored.printStackTrace();
}
if(s.length() == 0) {
Log.w("NET TOOLS", "string is empty");
return null;
} else {
//Log.d("NET TOOLS", s);
return s;
}
}
// https://stackoverflow.com/a/5445161
static String streamToString(InputStream is) {
Scanner s = new Scanner(is, "UTF-8").useDelimiter("\\A");
return s.hasNext() ? s.next() : "";
}
/**
* New method, maybe faster, to read inputStream
* also see https://stackoverflow.com/a/5445161
* @param is what to read
* @return the String Read
* @throws IOException from the InputStreamReader
*/
static String parseStreamToString(InputStream is) throws IOException{
final int bufferSize = 1024;
final char[] buffer = new char[bufferSize];
final StringBuilder out = new StringBuilder();
InputStreamReader in = new InputStreamReader(is, "UTF-8");
int rsz= in.read(buffer, 0, buffer.length);
while( rsz >0) {
out.append(buffer, 0, rsz);
rsz = in.read(buffer, 0, buffer.length);
}
return out.toString();
}
static int failsafeParseInt(String str) {
try {
return Integer.parseInt(str);
} catch(NumberFormatException e) {
return 0;
}
}
}
diff --git a/src/it/reyboz/bustorino/data/gtfs/CsvTableInserter.kt b/src/it/reyboz/bustorino/data/gtfs/CsvTableInserter.kt
index 0bf96e4..453630e 100644
--- a/src/it/reyboz/bustorino/data/gtfs/CsvTableInserter.kt
+++ b/src/it/reyboz/bustorino/data/gtfs/CsvTableInserter.kt
@@ -1,82 +1,115 @@
/*
BusTO - Data components
Copyright (C) 2021 Fabio Mazza
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see .
*/
package it.reyboz.bustorino.data.gtfs
import android.content.Context
import android.util.Log
import java.util.ArrayList
class CsvTableInserter(
val tableName: String, context: Context
) {
private val database: GtfsDatabase = GtfsDatabase.getGtfsDatabase(context)
private val dao: StaticGtfsDao = database.gtfsDao()
private val elementsList: MutableList< in GtfsTable> = mutableListOf()
+ private var stopsIDsPresent: HashSet? = null
+ private var tripsIDsPresent: HashSet? = null
+
+ private var countInsert = 0
+ init {
+ if(tableName == "stop_times") {
+ stopsIDsPresent = dao.getAllStopsIDs().toHashSet()
+ tripsIDsPresent = dao.getAllTripsIDs().toHashSet()
+ Log.d(DEBUG_TAG, "num stop IDs present: "+ stopsIDsPresent!!.size)
+ Log.d(DEBUG_TAG, "num trips IDs present: "+ tripsIDsPresent!!.size)
+ } else if(tableName == "routes"){
+ dao.deleteAllRoutes()
+ }
+ }
+
fun addElement(csvLineElements: Map) {
when(tableName){
"stops" ->
elementsList.add(GtfsStop(csvLineElements))
"routes" ->
elementsList.add(GtfsRoute(csvLineElements))
"calendar" ->
elementsList.add(GtfsService(csvLineElements))
"calendar_dates" ->
elementsList.add(GtfsServiceDate(csvLineElements))
"trips" ->
elementsList.add(GtfsTrip(csvLineElements))
"shapes" ->
elementsList.add(GtfsShape(csvLineElements))
- "stop_times" ->
- elementsList.add(GtfsStopTime(csvLineElements))
+ "stop_times" -> {
+ //filter stop
+ val stopTime = GtfsStopTime(csvLineElements)
+ /*
+ val stopOk = //tripsIDsPresent?.contains(stopTime.tripID) == true
+ (stopsIDsPresent?.contains(stopTime.stopID) == true)// &&
+ // tripsIDsPresent?.contains(stopTime.tripID) == true)
+ if (stopOk)
+ */
+ elementsList.add(stopTime)
+ }
}
if(elementsList.size >= MAX_ELEMENTS){
//have to insert
- Log.d(DEBUG_TAG, "Inserting first batch of elements now, list size: "+elementsList.size)
+
if (tableName == "routes")
dao.insertRoutes(elementsList.filterIsInstance())
else
insertDataInDatabase()
elementsList.clear()
}
}
- fun insertDataInDatabase(){
+ private fun insertDataInDatabase(){
+ //Log.d(DEBUG_TAG, "Inserting batch of elements now, list size: "+elementsList.size)
+ countInsert += elementsList.size
when(tableName){
- "stops" -> dao.updateStops(elementsList.filterIsInstance())
- "routes" -> dao.clearAndInsertRoutes(elementsList.filterIsInstance())
+ "stops" -> {
+ dao.insertStops(elementsList.filterIsInstance())
+ }
+ "routes" -> dao.insertRoutes(elementsList.filterIsInstance())
"calendar" -> dao.insertServices(elementsList.filterIsInstance())
"calendar_dates" -> dao.insertDates(elementsList.filterIsInstance())
"trips" -> dao.insertTrips(elementsList.filterIsInstance())
"stop_times"-> dao.insertStopTimes(elementsList.filterIsInstance())
"shapes" -> dao.insertShapes(elementsList.filterIsInstance())
}
+ ///if(elementsList.size < MAX_ELEMENTS)
+ }
+ fun finishInsert(){
+ insertDataInDatabase()
+ Log.d(DEBUG_TAG, "Inserted "+countInsert+" elements from "+tableName);
}
companion object{
- val MAX_ELEMENTS = 5000
+ const val MAX_ELEMENTS = 5000
- val DEBUG_TAG="BusTO - TableInserter"
+ const val DEBUG_TAG="BusTO - TableInserter"
}
}
\ No newline at end of file
diff --git a/src/it/reyboz/bustorino/data/gtfs/GtfsDatabase.kt b/src/it/reyboz/bustorino/data/gtfs/GtfsDatabase.kt
index 4ac09c6..986f01f 100644
--- a/src/it/reyboz/bustorino/data/gtfs/GtfsDatabase.kt
+++ b/src/it/reyboz/bustorino/data/gtfs/GtfsDatabase.kt
@@ -1,57 +1,57 @@
/*
BusTO - Data components
Copyright (C) 2021 Fabio Mazza
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see .
*/
package it.reyboz.bustorino.data.gtfs
import android.content.Context
import androidx.room.*
@Database(
entities = [
GtfsServiceDate::class,
GtfsStop::class,
GtfsService::class,
GtfsRoute::class,
GtfsStopTime::class,
GtfsTrip::class,
GtfsShape::class],
version = GtfsDatabase.VERSION,
exportSchema = false,
)
@TypeConverters(Converters::class)
public abstract class GtfsDatabase : RoomDatabase() {
abstract fun gtfsDao() : StaticGtfsDao
companion object{
@Volatile
private var INSTANCE: GtfsDatabase? =null
fun getGtfsDatabase(context: Context): GtfsDatabase{
return INSTANCE ?: synchronized(this){
val instance = Room.databaseBuilder(context.applicationContext,
GtfsDatabase::class.java,
"gtfs_database").build()
INSTANCE = instance
instance
}
}
const val VERSION = 1
- const val FOREIGNKEY_ONDELETE = ForeignKey.NO_ACTION
+ const val FOREIGNKEY_ONDELETE = ForeignKey.CASCADE
}
}
\ No newline at end of file
diff --git a/src/it/reyboz/bustorino/data/gtfs/GtfsTrip.kt b/src/it/reyboz/bustorino/data/gtfs/GtfsTrip.kt
index cba9c27..daf58eb 100644
--- a/src/it/reyboz/bustorino/data/gtfs/GtfsTrip.kt
+++ b/src/it/reyboz/bustorino/data/gtfs/GtfsTrip.kt
@@ -1,107 +1,107 @@
/*
BusTO - Data components
Copyright (C) 2021 Fabio Mazza
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see .
*/
package it.reyboz.bustorino.data.gtfs
import androidx.room.*
@Entity(tableName = GtfsTrip.DB_TABLE,
foreignKeys=[
ForeignKey(entity = GtfsRoute::class,
parentColumns = [GtfsRoute.COL_ROUTE_ID],
childColumns = [GtfsTrip.COL_ROUTE_ID],
- onDelete = GtfsDatabase.FOREIGNKEY_ONDELETE),
+ onDelete = ForeignKey.CASCADE),
// The service_id: ID referencing calendar.service_id or calendar_dates.service_id
/*
ForeignKey(entity = GtfsService::class,
parentColumns = [GtfsService.COL_SERVICE_ID],
childColumns = [GtfsTrips.COL_SERVICE_ID],
onDelete = GtfsDatabase.FOREIGNKEY_ONDELETE),
*/
],
indices = [Index(GtfsTrip.COL_ROUTE_ID)]
)
data class GtfsTrip(
@ColumnInfo(name = COL_ROUTE_ID )
val routeID: String,
@ColumnInfo(name = COL_SERVICE_ID)
val serviceID: String,
@PrimaryKey
@ColumnInfo(name = COL_TRIP_ID)
val tripID: String,
@ColumnInfo(name = COL_HEADSIGN)
val tripHeadsign: String,
@ColumnInfo(name = COL_DIRECTION_ID)
val directionID: Int,
@ColumnInfo(name = COL_BLOCK_ID)
val blockID: String,
@ColumnInfo(name = COL_SHAPE_ID)
val shapeID: String,
@ColumnInfo(name = COL_WHEELCHAIR)
val isWheelchairAccess: Boolean,
@ColumnInfo(name = COL_LIMITED_R)
val isLimitedRoute: Boolean,
): GtfsTable {
constructor(valuesByColumn: Map) : this(
valuesByColumn[COL_ROUTE_ID]!!,
valuesByColumn[COL_SERVICE_ID]!!,
valuesByColumn[COL_TRIP_ID]!!,
valuesByColumn[COL_HEADSIGN]!!,
valuesByColumn[COL_DIRECTION_ID]?.toIntOrNull()?: 0,
valuesByColumn[COL_BLOCK_ID]!!,
valuesByColumn[COL_SHAPE_ID]!!,
Converters.fromStringNum(valuesByColumn[COL_WHEELCHAIR], false),
Converters.fromStringNum(valuesByColumn[COL_LIMITED_R], false)
)
companion object{
const val DB_TABLE="gtfs_trips"
const val COL_ROUTE_ID="route_id"
const val COL_SERVICE_ID="service_id"
const val COL_TRIP_ID = "trip_id"
const val COL_HEADSIGN="trip_headsign"
//const val COL_SHORT_NAME="trip_short_name",
const val COL_DIRECTION_ID="direction_id"
const val COL_BLOCK_ID="block_id"
const val COL_SHAPE_ID = "shape_id"
const val COL_WHEELCHAIR="wheelchair_accessible"
const val COL_LIMITED_R="limited_route"
val COLUMNS= arrayOf(
COL_ROUTE_ID,
COL_SERVICE_ID,
COL_TRIP_ID,
COL_HEADSIGN,
COL_DIRECTION_ID,
COL_BLOCK_ID,
COL_SHAPE_ID,
COL_WHEELCHAIR,
COL_LIMITED_R
)
/*
open fun fromContentValues(values: ContentValues) {
val tripItem = GtfsTrips();
}
*/
}
override fun getColumns(): Array {
return COLUMNS
}
}
\ No newline at end of file
diff --git a/src/it/reyboz/bustorino/data/gtfs/StaticGtfsDao.kt b/src/it/reyboz/bustorino/data/gtfs/StaticGtfsDao.kt
index 86c0c70..32f862c 100644
--- a/src/it/reyboz/bustorino/data/gtfs/StaticGtfsDao.kt
+++ b/src/it/reyboz/bustorino/data/gtfs/StaticGtfsDao.kt
@@ -1,77 +1,90 @@
/*
BusTO - Data components
Copyright (C) 2021 Fabio Mazza
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see .
*/
package it.reyboz.bustorino.data.gtfs
import androidx.lifecycle.LiveData
import androidx.room.*
@Dao
interface StaticGtfsDao {
@Query("SELECT * FROM "+GtfsRoute.DB_TABLE+" ORDER BY "+GtfsRoute.COL_SORT_ORDER)
fun getAllRoutes() : LiveData>
+ @Query("SELECT "+GtfsTrip.COL_TRIP_ID+" FROM "+GtfsTrip.DB_TABLE)
+ fun getAllTripsIDs() : List
+
+ @Query("SELECT "+GtfsStop.COL_STOP_ID+" FROM "+GtfsStop.DB_TABLE)
+ fun getAllStopsIDs() : List
+
@Query("SELECT * FROM "+GtfsStop.DB_TABLE+" WHERE "+GtfsStop.COL_STOP_CODE+" LIKE :queryID")
fun getStopByStopID(queryID: String): LiveData>
@Query("SELECT * FROM "+GtfsShape.DB_TABLE+
" WHERE "+GtfsShape.COL_SHAPE_ID+" LIKE :shapeID"+
" ORDER BY "+GtfsShape.COL_POINT_SEQ+ " ASC"
)
fun getShapeByID(shapeID: String) : LiveData>
@Transaction
fun clearAndInsertRoutes(routes: List){
deleteAllRoutes()
insertRoutes(routes)
}
@Insert(onConflict = OnConflictStrategy.REPLACE)
fun insertRoutes(users: List)
- @Insert
+ @Insert(onConflict = OnConflictStrategy.REPLACE)
fun insertStops(stops: List)
@Insert(onConflict = OnConflictStrategy.REPLACE)
fun insertCalendarServices(services: List)
@Insert(onConflict = OnConflictStrategy.REPLACE)
fun insertShapes(shapes: List)
@Insert(onConflict = OnConflictStrategy.REPLACE)
fun insertDates(dates: List)
@Insert(onConflict = OnConflictStrategy.REPLACE)
fun insertServices(services: List)
@Insert(onConflict = OnConflictStrategy.REPLACE)
fun insertTrips(trips: List)
@Insert(onConflict = OnConflictStrategy.REPLACE)
fun insertStopTimes(stopTimes: List)
@Query("DELETE FROM "+GtfsRoute.DB_TABLE)
fun deleteAllRoutes()
@Query("DELETE FROM "+GtfsStop.DB_TABLE)
fun deleteAllStops()
+ @Query("DELETE FROM "+GtfsTrip.DB_TABLE)
+ fun deleteAllTrips()
@Update(onConflict = OnConflictStrategy.REPLACE)
fun updateShapes(shapes: List) : Int
@Transaction
- fun updateStops(stops: List){
+ fun updateAllStops(stops: List){
deleteAllStops()
insertStops(stops)
}
+ @Query("DELETE FROM "+GtfsStopTime.DB_TABLE)
+ fun deleteAllStopTimes()
+ @Query("DELETE FROM "+GtfsService.DB_TABLE)
+ fun deleteAllServices()
+
}
\ No newline at end of file