diff --git a/app/src/org/commcare/CommCareApplication.java b/app/src/org/commcare/CommCareApplication.java index 3a867aad12..9cef2143c7 100644 --- a/app/src/org/commcare/CommCareApplication.java +++ b/app/src/org/commcare/CommCareApplication.java @@ -73,7 +73,7 @@ import org.commcare.models.database.MigrationException; import org.commcare.models.database.SqlStorage; import org.commcare.models.database.global.DatabaseGlobalOpenHelper; -import org.commcare.models.database.user.models.EntityStorageCache; +import org.commcare.models.database.user.models.CommCareEntityStorageCache; import org.commcare.models.legacy.LegacyInstallUtils; import org.commcare.modern.database.Table; import org.commcare.modern.session.SessionWrapper; @@ -780,8 +780,8 @@ public void onServiceConnected(ComponentName className, IBinder service) { PurgeStaleArchivedFormsTask.launchPurgeTask(); } - if (EntityStorageCache.getEntityCacheWipedPref(user.getUniqueId()) < ReportingUtils.getAppVersion()) { - EntityStorageCache.wipeCacheForCurrentApp(); + if (CommCareEntityStorageCache.getEntityCacheWipedPref(user.getUniqueId()) < ReportingUtils.getAppVersion()) { + CommCareEntityStorageCache.wipeCacheForCurrentApp(); } purgeLogs(); diff --git a/app/src/org/commcare/adapters/EntityListAdapter.java b/app/src/org/commcare/adapters/EntityListAdapter.java index 3a7bf770c5..bcf6d2c5ba 100644 --- a/app/src/org/commcare/adapters/EntityListAdapter.java +++ b/app/src/org/commcare/adapters/EntityListAdapter.java @@ -1,5 +1,7 @@ package org.commcare.adapters; +import static org.commcare.cases.util.StringUtils.normalize; + import android.database.DataSetObserver; import android.view.LayoutInflater; import android.view.View; @@ -20,7 +22,6 @@ import org.commcare.suite.model.Detail; import org.commcare.utils.AndroidUtil; import org.commcare.utils.CachingAsyncImageLoader; -import org.commcare.utils.StringUtils; import org.commcare.views.EntityActionViewUtils; import org.commcare.views.EntityView; import org.commcare.views.EntityViewTile; @@ -311,7 +312,7 @@ public synchronized void filterByString(String filterRaw) { // split by whitespace String[] searchTerms = filterRaw.split("\\s+"); for (int i = 0; i < searchTerms.length; ++i) { - searchTerms[i] = StringUtils.normalize(searchTerms[i]); + searchTerms[i] = normalize(searchTerms[i]); } currentSearchTerms = searchTerms; searchQuery = filterRaw; diff --git a/app/src/org/commcare/interfaces/AndroidSortableEntityAdapter.java b/app/src/org/commcare/interfaces/AndroidSortableEntityAdapter.java index 25d82b6f58..91873891af 100644 --- a/app/src/org/commcare/interfaces/AndroidSortableEntityAdapter.java +++ b/app/src/org/commcare/interfaces/AndroidSortableEntityAdapter.java @@ -1,10 +1,10 @@ package org.commcare.interfaces; import org.commcare.CommCareApplication; +import org.commcare.cases.entity.AsyncNodeEntityFactory; import org.commcare.cases.entity.Entity; import org.commcare.cases.entity.NodeEntityFactory; import org.commcare.cases.entity.SortableEntityAdapter; -import org.commcare.models.AsyncNodeEntityFactory; import org.commcare.suite.model.Detail; import org.commcare.views.notifications.NotificationMessageFactory; import org.javarosa.core.model.instance.TreeReference; diff --git a/app/src/org/commcare/models/AsyncEntity.java b/app/src/org/commcare/models/AsyncEntity.java deleted file mode 100755 index 7a8b850ffd..0000000000 --- a/app/src/org/commcare/models/AsyncEntity.java +++ /dev/null @@ -1,254 +0,0 @@ -package org.commcare.models; - -import net.sqlcipher.database.SQLiteDatabase; - -import org.commcare.CommCareApplication; -import org.commcare.cases.entity.Entity; -import org.commcare.logging.XPathErrorLogger; -import org.commcare.models.database.user.models.EntityStorageCache; -import org.commcare.suite.model.DetailField; -import org.commcare.suite.model.Text; -import org.commcare.utils.SessionUnavailableException; -import org.commcare.utils.StringUtils; -import org.javarosa.core.model.condition.EvaluationContext; -import org.javarosa.core.model.instance.TreeReference; -import org.javarosa.xpath.XPathException; -import org.javarosa.xpath.expr.FunctionUtils; -import org.javarosa.xpath.expr.XPathExpression; -import org.javarosa.xpath.parser.XPathSyntaxException; - -import java.util.Enumeration; -import java.util.Hashtable; - -/** - * An AsyncEntity is an entity reference which is capable of building its - * values (evaluating all Text elements/background data elements) lazily - * rather than upfront when the entity is constructed. - * - * It is threadsafe. - * - * It will attempt to Cache its values persistently by a derived entity key rather - * than evaluating them each time when possible. This can be slow to perform across - * all entities internally due to the overhead of establishing the db connection, it - * is recommended that the entities be primed externally with a bulk query. - * - * @author ctsims - */ -public class AsyncEntity extends Entity { - - private final DetailField[] fields; - private final Object[] data; - private final String[] sortData; - private final boolean[] relevancyData; - private final String[][] sortDataPieces; - private final EvaluationContext context; - private final Hashtable mVariableDeclarations; - private boolean mVariableContextLoaded = false; - private final String mCacheIndex; - private final String mDetailId; - - private final EntityStorageCache mEntityStorageCache; - - /* - * the Object's lock. NOTE: _DO NOT LOCK ANY CODE WHICH READS/WRITES THE CACHE - * UNTIL YOU HAVE A LOCK FOR THE DB! - * - * The lock is for the integrity of this object, not the larger environment, - * and any DB access has its own implict lock between threads, so it's easy - * to accidentally deadlock if you don't already have the db lock - * - * Basically you should never be calling mEntityStorageCache from inside of - * a lock that - */ - private final Object mAsyncLock = new Object(); - - public AsyncEntity(DetailField[] fields, EvaluationContext ec, - TreeReference t, Hashtable variables, - EntityStorageCache cache, String cacheIndex, String detailId, - String extraKey) { - super(t, extraKey); - - this.fields = fields; - this.data = new Object[fields.length]; - this.sortData = new String[fields.length]; - this.sortDataPieces = new String[fields.length][]; - this.relevancyData = new boolean[fields.length]; - this.context = ec; - this.mVariableDeclarations = variables; - this.mEntityStorageCache = cache; - - //TODO: It's weird that we pass this in, kind of, but the thing is that we don't want to figure out - //if this ref is _cachable_ every time, since it's a pretty big lift - this.mCacheIndex = cacheIndex; - - this.mDetailId = detailId; - } - - private void loadVariableContext() { - synchronized (mAsyncLock) { - if (!mVariableContextLoaded) { - //These are actually in an ordered hashtable, so we can't just get the keyset, since it's - //in a 1.3 hashtable equivalent - for (Enumeration en = mVariableDeclarations.keys(); en.hasMoreElements(); ) { - String key = en.nextElement(); - context.setVariable(key, FunctionUtils.unpack(mVariableDeclarations.get(key).eval(context))); - } - mVariableContextLoaded = true; - } - } - } - - @Override - public Object getField(int i) { - synchronized (mAsyncLock) { - loadVariableContext(); - if (data[i] == null) { - try { - data[i] = fields[i].getTemplate().evaluate(context); - } catch (XPathException xpe) { - XPathErrorLogger.INSTANCE.logErrorToCurrentApp(xpe); - xpe.printStackTrace(); - data[i] = ""; - } - } - return data[i]; - } - } - - @Override - public String getNormalizedField(int i) { - String normalized = this.getSortField(i); - if (normalized == null) { - return ""; - } - return normalized; - } - - @Override - public String getSortField(int i) { - //Get a db handle so we can get an outer lock - SQLiteDatabase db; - try { - db = CommCareApplication.instance().getUserDbHandle(); - } catch (SessionUnavailableException e) { - return null; - } - - //get the db lock - db.beginTransaction(); - try { - //get our second lock. - synchronized (mAsyncLock) { - if (sortData[i] == null) { - // sort data not in search field cache; load and store it - Text sortText = fields[i].getSort(); - if (sortText == null) { - db.setTransactionSuccessful(); - return null; - } - - String cacheKey = AsyncNodeEntityFactory.getCacheKey(mDetailId, String.valueOf(i)); - - if (mCacheIndex != null) { - //Check the cache! - String value = mEntityStorageCache.retrieveCacheValue(mCacheIndex, cacheKey); - if (value != null) { - this.setSortData(i, value); - db.setTransactionSuccessful(); - return sortData[i]; - } - } - - loadVariableContext(); - try { - sortText = fields[i].getSort(); - if (sortText == null) { - this.setSortData(i, getFieldString(i)); - } else { - this.setSortData(i, StringUtils.normalize(sortText.evaluate(context))); - } - - mEntityStorageCache.cache(mCacheIndex, cacheKey, sortData[i]); - } catch (XPathException xpe) { - XPathErrorLogger.INSTANCE.logErrorToCurrentApp(xpe); - xpe.printStackTrace(); - sortData[i] = ""; - } - } - db.setTransactionSuccessful(); - return sortData[i]; - } - - } finally { - //free the db lock. - db.endTransaction(); - } - } - - @Override - public int getNumFields() { - return fields.length; - } - - @Override - public boolean isValidField(int fieldIndex) { - //NOTE: This totally jacks the asynchronicity. It's only used in - //detail fields for now, so not super important, but worth bearing - //in mind - synchronized (mAsyncLock) { - loadVariableContext(); - if (getField(fieldIndex).equals("")) { - return false; - } - - try { - this.relevancyData[fieldIndex] = this.fields[fieldIndex].isRelevant(this.context); - } catch (XPathSyntaxException e) { - final String msg = "Invalid relevant condition for field : " + fields[fieldIndex].getHeader().toString(); - XPathErrorLogger.INSTANCE.logErrorToCurrentApp(msg); - throw new RuntimeException(msg); - } - return this.relevancyData[fieldIndex]; - } - } - - @Override - public Object[] getData() { - for (int i = 0; i < this.getNumFields(); ++i) { - this.getField(i); - } - return data; - } - - @Override - public String[] getSortFieldPieces(int i) { - if (getSortField(i) == null) { - return new String[0]; - } - return sortDataPieces[i]; - } - - private void setSortData(int i, String val) { - synchronized (mAsyncLock) { - this.sortData[i] = val; - this.sortDataPieces[i] = breakUpField(val); - } - } - - public void setSortData(String cacheKey, String val) { - int sortIndex = EntityStorageCache.getSortFieldIdFromCacheKey(mDetailId, cacheKey); - if (sortIndex != -1) { - setSortData(sortIndex, val); - } - } - - private static String[] breakUpField(String input) { - if (input == null) { - return new String[0]; - } else { - //We always fuzzy match on the sort field and only if it is available - //(as a way to restrict possible matching) - return input.split("\\s+"); - } - } -} diff --git a/app/src/org/commcare/models/AsyncNodeEntityFactory.java b/app/src/org/commcare/models/AsyncNodeEntityFactory.java deleted file mode 100755 index f9c8cc320c..0000000000 --- a/app/src/org/commcare/models/AsyncNodeEntityFactory.java +++ /dev/null @@ -1,231 +0,0 @@ -package org.commcare.models; - -import android.util.Log; - -import net.sqlcipher.Cursor; -import net.sqlcipher.database.SQLiteDatabase; - -import org.commcare.AppUtils; -import org.commcare.CommCareApplication; -import org.commcare.android.logging.ReportingUtils; -import org.commcare.cases.entity.Entity; -import org.commcare.cases.entity.NodeEntityFactory; -import org.commcare.models.database.DbUtil; -import org.commcare.models.database.SqlStorage; -import org.commcare.models.database.user.models.EntityStorageCache; -import org.commcare.modern.database.TableBuilder; -import org.commcare.suite.model.Detail; -import org.commcare.suite.model.DetailField; -import org.javarosa.core.model.condition.EvaluationContext; -import org.javarosa.core.model.instance.TreeReference; -import org.javarosa.core.model.utils.CacheHost; -import org.javarosa.core.util.OrderedHashtable; -import org.javarosa.xpath.expr.XPathExpression; - -import java.util.Hashtable; -import java.util.List; -import java.util.Vector; - -/** - * @author ctsims - */ -public class AsyncNodeEntityFactory extends NodeEntityFactory { - private static final String TAG = AsyncNodeEntityFactory.class.getSimpleName(); - private final OrderedHashtable mVariableDeclarations; - - private final Hashtable mEntitySet = new Hashtable<>(); - private final EntityStorageCache mEntityCache; - - private CacheHost mCacheHost = null; - private Boolean mTemplateIsCachable = null; - private static final Object mAsyncLock = new Object(); - private Thread mAsyncPrimingThread; - - // Don't show entity list until we primeCache and caches all fields - private final boolean isBlockingAsyncMode; - - public AsyncNodeEntityFactory(Detail d, EvaluationContext ec) { - super(d, ec); - - mVariableDeclarations = detail.getVariableDeclarations(); - mEntityCache = new EntityStorageCache("case"); - isBlockingAsyncMode = detail.hasSortField(); - } - - @Override - public Entity getEntity(TreeReference data) { - EvaluationContext nodeContext = new EvaluationContext(ec, data); - - mCacheHost = nodeContext.getCacheHost(data); - - String mCacheIndex = null; - if (mTemplateIsCachable == null) { - mTemplateIsCachable = mCacheHost != null && mCacheHost.isReferencePatternCachable(data); - } - if (mTemplateIsCachable) { - if (mCacheHost == null) { - Log.d(TAG, "Template is cachable, but there's no cache host for this instance?"); - } else { - mCacheIndex = mCacheHost.getCacheIndex(data); - } - } - - String entityKey = loadCalloutDataMapKey(nodeContext); - AsyncEntity entity = - new AsyncEntity(detail.getFields(), nodeContext, data, mVariableDeclarations, - mEntityCache, mCacheIndex, detail.getId(), entityKey); - - if (mCacheIndex != null) { - mEntitySet.put(mCacheIndex, entity); - } - return entity; - } - - @Override - protected void setEvaluationContextDefaultQuerySet(EvaluationContext ec, - List result) { - - //Don't do anything for asynchronous lists. In theory the query set could help expand the - //first cache more quickly, but otherwise it's just keeping around tons of cases in memory - //that don't even need to be loaded. - } - - - /** - * Bulk loads search field cache from db. - * Note that the cache is lazily built upon first case list search. - */ - private void primeCache() { - if (mTemplateIsCachable == null || !mTemplateIsCachable || mCacheHost == null) { - return; - } - - String[][] cachePrimeKeys = mCacheHost.getCachePrimeGuess(); - if (cachePrimeKeys == null) { - return; - } - - Vector sortKeys = new Vector<>(); - String validKeys = buildValidKeys(sortKeys, detail.getFields()); - if ("".equals(validKeys)) { - return; - } - - //Create our full args tree. We need the elements from the cache primer - //along with the specific keys we wanna pull out - - String[] args = new String[cachePrimeKeys[1].length + sortKeys.size()]; - System.arraycopy(cachePrimeKeys[1], 0, args, 0, cachePrimeKeys[1].length); - - for (int i = 0; i < sortKeys.size(); ++i) { - args[cachePrimeKeys[1].length + i] = getCacheKey(detail.getId(), String.valueOf(sortKeys.get(i))); - } - - String[] names = cachePrimeKeys[0]; - String whereClause = buildKeyNameWhereClause(names); - - long now = System.currentTimeMillis(); - - SQLiteDatabase db = CommCareApplication.instance().getUserDbHandle(); - - String sqlStatement = "SELECT entity_key, cache_key, value FROM entity_cache JOIN AndroidCase ON entity_cache.entity_key = AndroidCase.commcare_sql_id WHERE " + - whereClause + " AND " + EntityStorageCache.COL_APP_ID + " = '" + AppUtils.getCurrentAppId() + - "' AND cache_key IN " + validKeys; - if (SqlStorage.STORAGE_OUTPUT_DEBUG) { - DbUtil.explainSql(db, sqlStatement, args); - } - - populateEntitySet(db, sqlStatement, args); - - if (SqlStorage.STORAGE_OUTPUT_DEBUG) { - Log.d(TAG, "Sequential Cache Load: " + (System.currentTimeMillis() - now) + "ms"); - } - } - - private String buildValidKeys(Vector sortKeys, DetailField[] fields) { - String validKeys = "("; - boolean added = false; - for (int i = 0; i < fields.length; ++i) { - //We're only gonna pull out the fields we can index/sort on - if (fields[i].getSort() != null) { - sortKeys.add(i); - validKeys += "?, "; - added = true; - } - } - if (added) { - return validKeys.substring(0, validKeys.length() - 2) + ")"; - } else { - return ""; - } - } - - public static String getCacheKey(String detailId, String mFieldId) { - return detailId + "_" + mFieldId; - } - - private String buildKeyNameWhereClause(String[] names) { - String whereClause = ""; - for (int i = 0; i < names.length; ++i) { - whereClause += TableBuilder.scrubName(names[i]) + " = ?"; - if (i + 1 < names.length) { - whereClause += " AND "; - } - } - return whereClause; - } - - private void populateEntitySet(SQLiteDatabase db, String sqlStatement, String[] args) { - //TODO: This will _only_ query up to about a meg of data, which is an un-great limitation. - //Should probably split this up SQL LIMIT based looped - //For reference the current limitation is about 10k rows with 1 field each. - Cursor walker = db.rawQuery(sqlStatement, args); - while (walker.moveToNext()) { - String entityId = walker.getString(walker.getColumnIndex("entity_key")); - String cacheId = walker.getString(walker.getColumnIndex("cache_key")); - String val = walker.getString(walker.getColumnIndex("value")); - if (this.mEntitySet.containsKey(entityId)) { - this.mEntitySet.get(entityId).setSortData(cacheId, val); - } - } - walker.close(); - } - - @Override - protected void prepareEntitiesInternal(List> entities) { - // if blocking mode load cache on the same thread and set any data thats not cached - if (isBlockingAsyncMode) { - primeCache(); - setUnCachedData(entities); - } else { - // otherwise we want to show the entity list asap and hence want to offload the loading cache part to a separate - // thread while caching any uncached data later on UI thread during Adapter's getView - synchronized (mAsyncLock) { - if (mAsyncPrimingThread == null) { - mAsyncPrimingThread = new Thread(this::primeCache); - mAsyncPrimingThread.start(); - } - } - } - } - - private void setUnCachedData(List> entities) { - for (int i = 0; i < entities.size(); i++) { - AsyncEntity e = (AsyncEntity)entities.get(i); - for (int col = 0; col < e.getNumFields(); ++col) { - e.getSortField(col); - } - } - } - - @Override - protected boolean isEntitySetReadyInternal() { - synchronized (mAsyncLock) { - return mAsyncPrimingThread == null || !mAsyncPrimingThread.isAlive(); - } - } - - public boolean isBlockingAsyncMode() { - return isBlockingAsyncMode; - } -} diff --git a/app/src/org/commcare/models/database/user/DatabaseUserOpenHelper.java b/app/src/org/commcare/models/database/user/DatabaseUserOpenHelper.java index a9560f0931..ff0ea70c0c 100755 --- a/app/src/org/commcare/models/database/user/DatabaseUserOpenHelper.java +++ b/app/src/org/commcare/models/database/user/DatabaseUserOpenHelper.java @@ -20,7 +20,7 @@ import org.commcare.models.database.IndexedFixturePathUtils; import org.commcare.android.database.user.models.ACase; import org.commcare.models.database.user.models.AndroidCaseIndexTable; -import org.commcare.models.database.user.models.EntityStorageCache; +import org.commcare.models.database.user.models.CommCareEntityStorageCache; import org.commcare.android.database.user.models.FormRecord; import org.commcare.android.database.user.models.SessionStateDescriptor; import org.commcare.modern.database.DatabaseIndexingUtils; @@ -162,8 +162,8 @@ public void onCreate(SQLiteDatabase database) { DbUtil.createNumbersTable(database); - database.execSQL(EntityStorageCache.getTableDefinition()); - EntityStorageCache.createIndexes(database); + database.execSQL(CommCareEntityStorageCache.getTableDefinition()); + CommCareEntityStorageCache.createIndexes(database); database.execSQL(AndroidCaseIndexTable.getTableDefinition()); AndroidCaseIndexTable.createIndexes(database); diff --git a/app/src/org/commcare/models/database/user/UserDatabaseUpgrader.java b/app/src/org/commcare/models/database/user/UserDatabaseUpgrader.java index ee68f309a7..895ac83ff5 100755 --- a/app/src/org/commcare/models/database/user/UserDatabaseUpgrader.java +++ b/app/src/org/commcare/models/database/user/UserDatabaseUpgrader.java @@ -30,7 +30,7 @@ import org.commcare.android.database.user.models.ACasePreV6Model; import org.commcare.android.database.user.models.AUser; import org.commcare.models.database.user.models.AndroidCaseIndexTable; -import org.commcare.models.database.user.models.EntityStorageCache; +import org.commcare.models.database.user.models.CommCareEntityStorageCache; import org.commcare.android.database.user.models.FormRecord; import org.commcare.android.database.user.models.FormRecordV1; import org.commcare.android.database.user.models.GeocodeCacheModel; @@ -283,8 +283,8 @@ private boolean upgradeFiveSix(SQLiteDatabase db) { db.execSQL(DatabaseIndexingUtils.indexOnTableCommand("case_status_open_index", "AndroidCase", "case_type,case_status")); DbUtil.createNumbersTable(db); - db.execSQL(EntityStorageCache.getTableDefinition()); - EntityStorageCache.createIndexes(db); + db.execSQL(CommCareEntityStorageCache.getTableDefinition()); + CommCareEntityStorageCache.createIndexes(db); db.execSQL(AndroidCaseIndexTablePreV21.getTableDefinition()); AndroidCaseIndexTable.createIndexes(db); @@ -626,8 +626,8 @@ private boolean upgradeTwentyOneTwentyTwo(SQLiteDatabase db) { //drop the existing table and recreate using current definition db.beginTransaction(); try { - db.execSQL("DROP TABLE IF EXISTS " + EntityStorageCache.TABLE_NAME); - db.execSQL(EntityStorageCache.getTableDefinition()); + db.execSQL("DROP TABLE IF EXISTS " + CommCareEntityStorageCache.TABLE_NAME); + db.execSQL(CommCareEntityStorageCache.getTableDefinition()); db.setTransactionSuccessful(); return true; } finally { diff --git a/app/src/org/commcare/models/database/user/models/EntityStorageCache.java b/app/src/org/commcare/models/database/user/models/CommCareEntityStorageCache.java similarity index 56% rename from app/src/org/commcare/models/database/user/models/EntityStorageCache.java rename to app/src/org/commcare/models/database/user/models/CommCareEntityStorageCache.java index 4bbe38ea46..b0509bf35a 100755 --- a/app/src/org/commcare/models/database/user/models/EntityStorageCache.java +++ b/app/src/org/commcare/models/database/user/models/CommCareEntityStorageCache.java @@ -8,21 +8,29 @@ import org.commcare.AppUtils; import org.commcare.CommCareApplication; -import org.commcare.android.logging.ReportingUtils; -import org.commcare.modern.database.TableBuilder; +import org.commcare.cases.entity.AsyncEntity; +import org.commcare.cases.entity.EntityStorageCache; +import org.commcare.models.database.DbUtil; import org.commcare.models.database.SqlStorage; import org.commcare.modern.database.DatabaseHelper; import org.commcare.modern.database.DatabaseIndexingUtils; +import org.commcare.modern.database.TableBuilder; import org.commcare.modern.util.Pair; +import org.commcare.suite.model.Detail; +import org.commcare.suite.model.DetailField; +import org.commcare.utils.SessionUnavailableException; +import java.io.Closeable; import java.util.Collection; +import java.util.Hashtable; import java.util.List; +import java.util.Vector; /** * @author ctsims */ -public class EntityStorageCache { - private static final String TAG = EntityStorageCache.class.getSimpleName(); +public class CommCareEntityStorageCache implements EntityStorageCache { + private static final String TAG = CommCareEntityStorageCache.class.getSimpleName(); public static final String TABLE_NAME = "entity_cache"; public static final String COL_APP_ID = "app_id"; @@ -37,11 +45,11 @@ public class EntityStorageCache { private final String mCacheName; private final String mAppId; - public EntityStorageCache(String cacheName) { + public CommCareEntityStorageCache(String cacheName) { this(cacheName, CommCareApplication.instance().getUserDbHandle(), AppUtils.getCurrentAppId()); } - public EntityStorageCache(String cacheName, SQLiteDatabase db, String appId) { + public CommCareEntityStorageCache(String cacheName, SQLiteDatabase db, String appId) { this.db = db; this.mCacheName = cacheName; this.mAppId = appId; @@ -65,6 +73,17 @@ public static void createIndexes(SQLiteDatabase db) { db.execSQL(DatabaseIndexingUtils.indexOnTableCommand("NAME_ENTITY_KEY", TABLE_NAME, COL_CACHE_NAME + ", " + COL_ENTITY_KEY + ", " + COL_CACHE_KEY)); } + public Closeable lockCache() { + //Get a db handle so we can get an outer lock + SQLiteDatabase db = CommCareApplication.instance().getUserDbHandle(); + //get the db lock + db.beginTransaction(); + return () -> { + db.setTransactionSuccessful(); + db.endTransaction(); + }; + } + //TODO: We should do some synchronization to make it the case that nothing can hold //an object for the same cache at once @@ -125,7 +144,7 @@ public void invalidateCaches(Collection recordIds) { } - public static int getSortFieldIdFromCacheKey(String detailId, String cacheKey) { + public int getSortFieldIdFromCacheKey(String detailId, String cacheKey) { String intId = cacheKey.substring(detailId.length() + 1); try { return Integer.parseInt(intId); @@ -163,4 +182,90 @@ public static int getEntityCacheWipedPref(String uuid) { return CommCareApplication.instance().getCurrentApp().getAppPreferences() .getInt(uuid + "_" + ENTITY_CACHE_WIPED_PREF_SUFFIX, -1); } + + public void primeCache(Hashtable entitySet, String[][] cachePrimeKeys, + Detail detail) { + Vector sortKeys = new Vector<>(); + String validKeys = buildValidKeys(sortKeys, detail.getFields()); + if ("".equals(validKeys)) { + return; + } + + //Create our full args tree. We need the elements from the cache primer + //along with the specific keys we wanna pull out + + String[] args = new String[cachePrimeKeys[1].length + sortKeys.size()]; + System.arraycopy(cachePrimeKeys[1], 0, args, 0, cachePrimeKeys[1].length); + + for (int i = 0; i < sortKeys.size(); ++i) { + args[cachePrimeKeys[1].length + i] = getCacheKey(detail.getId(), String.valueOf(sortKeys.get(i))); + } + + String[] names = cachePrimeKeys[0]; + String whereClause = buildKeyNameWhereClause(names); + long now = System.currentTimeMillis(); + String sqlStatement = "SELECT entity_key, cache_key, value FROM entity_cache JOIN AndroidCase ON entity_cache.entity_key = AndroidCase.commcare_sql_id WHERE " + + whereClause + " AND " + CommCareEntityStorageCache.COL_APP_ID + " = '" + AppUtils.getCurrentAppId() + + "' AND cache_key IN " + validKeys; + SQLiteDatabase db = CommCareApplication.instance().getUserDbHandle(); + if (SqlStorage.STORAGE_OUTPUT_DEBUG) { + DbUtil.explainSql(db, sqlStatement, args); + } + + populateEntitySet(db, sqlStatement, args, entitySet); + + if (SqlStorage.STORAGE_OUTPUT_DEBUG) { + Log.d(TAG, "Sequential Cache Load: " + (System.currentTimeMillis() - now) + "ms"); + } + } + + public String getCacheKey(String detailId, String mFieldId) { + return detailId + "_" + mFieldId; + } + + private static String buildValidKeys(Vector sortKeys, DetailField[] fields) { + String validKeys = "("; + boolean added = false; + for (int i = 0; i < fields.length; ++i) { + //We're only gonna pull out the fields we can index/sort on + if (fields[i].getSort() != null) { + sortKeys.add(i); + validKeys += "?, "; + added = true; + } + } + if (added) { + return validKeys.substring(0, validKeys.length() - 2) + ")"; + } else { + return ""; + } + } + + private static String buildKeyNameWhereClause(String[] names) { + String whereClause = ""; + for (int i = 0; i < names.length; ++i) { + whereClause += TableBuilder.scrubName(names[i]) + " = ?"; + if (i + 1 < names.length) { + whereClause += " AND "; + } + } + return whereClause; + } + + private static void populateEntitySet(SQLiteDatabase db, String sqlStatement, String[] args, + Hashtable entitySet) { + //TODO: This will _only_ query up to about a meg of data, which is an un-great limitation. + //Should probably split this up SQL LIMIT based looped + //For reference the current limitation is about 10k rows with 1 field each. + Cursor walker = db.rawQuery(sqlStatement, args); + while (walker.moveToNext()) { + String entityId = walker.getString(walker.getColumnIndex("entity_key")); + String cacheId = walker.getString(walker.getColumnIndex("cache_key")); + String val = walker.getString(walker.getColumnIndex("value")); + if (entitySet.containsKey(entityId)) { + entitySet.get(entityId).setSortData(cacheId, val); + } + } + walker.close(); + } } diff --git a/app/src/org/commcare/tasks/DataPullTask.java b/app/src/org/commcare/tasks/DataPullTask.java index 6dbe8c1e22..7f5242b644 100644 --- a/app/src/org/commcare/tasks/DataPullTask.java +++ b/app/src/org/commcare/tasks/DataPullTask.java @@ -22,7 +22,7 @@ import org.commcare.interfaces.CommcareRequestEndpoints; import org.commcare.models.database.SqlStorage; import org.commcare.models.database.user.models.AndroidCaseIndexTable; -import org.commcare.models.database.user.models.EntityStorageCache; +import org.commcare.models.database.user.models.CommCareEntityStorageCache; import org.commcare.models.encryption.ByteEncrypter; import org.commcare.modern.models.RecordTooLargeException; import org.commcare.network.DataPullRequester; @@ -579,7 +579,7 @@ private void wipeStorageForFourTwelveSync(SQLiteDatabase userDb) { SqlStorage.wipeTableWithoutCommit(userDb, ACase.STORAGE_KEY); SqlStorage.wipeTableWithoutCommit(userDb, Ledger.STORAGE_KEY); SqlStorage.wipeTableWithoutCommit(userDb, AndroidCaseIndexTable.TABLE_NAME); - EntityStorageCache.wipeCacheForCurrentAppWithoutCommit(userDb); + CommCareEntityStorageCache.wipeCacheForCurrentAppWithoutCommit(userDb); } private void updateCurrentUser(String password) { diff --git a/app/src/org/commcare/tasks/EntityLoaderTask.java b/app/src/org/commcare/tasks/EntityLoaderTask.java index 13cbc6c3ca..3bbaec7adf 100644 --- a/app/src/org/commcare/tasks/EntityLoaderTask.java +++ b/app/src/org/commcare/tasks/EntityLoaderTask.java @@ -4,14 +4,15 @@ import org.commcare.activities.EntitySelectActivity; import org.commcare.android.logging.ForceCloseLogger; +import org.commcare.cases.entity.AsyncNodeEntityFactory; import org.commcare.cases.entity.Entity; +import org.commcare.cases.entity.EntityStorageCache; import org.commcare.cases.entity.NodeEntityFactory; import org.commcare.logging.XPathErrorLogger; -import org.commcare.models.AsyncNodeEntityFactory; +import org.commcare.models.database.user.models.CommCareEntityStorageCache; import org.commcare.preferences.DeveloperPreferences; import org.commcare.suite.model.Detail; import org.commcare.tasks.templates.ManagedAsyncTask; -import org.commcare.util.LogTypes; import org.javarosa.core.model.condition.EvaluationContext; import org.javarosa.core.model.instance.TreeReference; import org.javarosa.core.services.Logger; @@ -37,7 +38,8 @@ public class EntityLoaderTask public EntityLoaderTask(Detail detail, EvaluationContext evalCtx) { evalCtx.addFunctionHandler(EntitySelectActivity.getHereFunctionHandler()); if (detail.useAsyncStrategy()) { - this.factory = new AsyncNodeEntityFactory(detail, evalCtx); + EntityStorageCache entityStorageCache = new CommCareEntityStorageCache("case"); + this.factory = new AsyncNodeEntityFactory(detail, evalCtx, entityStorageCache); } else { this.factory = new NodeEntityFactory(detail, evalCtx); if (DeveloperPreferences.collectAndDisplayEntityTraces()) { diff --git a/app/src/org/commcare/utils/StringUtils.java b/app/src/org/commcare/utils/StringUtils.java index 4cba48eee5..bacc412eca 100755 --- a/app/src/org/commcare/utils/StringUtils.java +++ b/app/src/org/commcare/utils/StringUtils.java @@ -19,47 +19,6 @@ */ public class StringUtils { - //TODO: Bro you can't just cache every fucking string ever. - private static LruCache normalizationCache; - - private static Pattern diacritics; - - //TODO: Really not sure about this size. Also, the LRU probably isn't really the best model here - //since we'd _like_ for these caches to get cleaned up at _some_ point. - static final private int cacheSize = 100 * 1024; - - /** - * @param input A non-null string - * @return a canonical version of the passed in string that is lower cased and has removed diacritical marks - * like accents. - */ - @SuppressLint("NewApi") - public synchronized static String normalize(String input) { - if (normalizationCache == null) { - normalizationCache = new LruCache<>(cacheSize); - - diacritics = Pattern.compile("\\p{InCombiningDiacriticalMarks}+"); - } - String cachedString = normalizationCache.get(input); - if (cachedString != null) { - return cachedString; - } - - //Initialized the normalized string (If we can, we'll use the Normalizer API on it) - String normalized = input; - - //If we're above gingerbread we'll normalize this in NFD form - //which helps a lot. Otherwise we won't be able to clear up some of those - //issues, but we can at least still eliminate diacritics. - normalized = Normalizer.normalize(input, Normalizer.Form.NFD); - - String output = diacritics.matcher(normalized).replaceAll("").toLowerCase(); - - normalizationCache.put(input, output); - - return output; - } - public static String getStringRobust(Context c, int resId) { return getStringRobust(c, resId, ""); } diff --git a/app/src/org/commcare/views/EntityView.java b/app/src/org/commcare/views/EntityView.java index 34d77eed26..5d56cd7739 100755 --- a/app/src/org/commcare/views/EntityView.java +++ b/app/src/org/commcare/views/EntityView.java @@ -1,5 +1,7 @@ package org.commcare.views; +import static org.commcare.cases.util.StringUtils.normalize; + import android.content.Context; import android.graphics.Bitmap; import android.graphics.Color; @@ -15,18 +17,16 @@ import android.widget.TextView; import org.commcare.android.logging.ForceCloseLogger; +import org.commcare.cases.entity.AsyncEntity; import org.commcare.cases.entity.Entity; -import org.commcare.dalvik.R; import org.commcare.core.graph.model.GraphData; import org.commcare.core.graph.util.GraphException; +import org.commcare.dalvik.R; import org.commcare.graph.view.GraphView; -import org.commcare.models.AsyncEntity; import org.commcare.suite.model.Detail; import org.commcare.suite.model.DetailField; -import org.commcare.util.LogTypes; import org.commcare.utils.AndroidUtil; import org.commcare.utils.MediaUtil; -import org.commcare.utils.StringUtils; import org.commcare.views.media.AudioPlaybackButton; import org.commcare.views.media.ViewId; import org.javarosa.core.services.Logger; @@ -392,14 +392,14 @@ public static Spannable highlightSearches(String[] searchTerms, Spannable raw, } //make sure that we have the same consistency for our background match - backgroundString = StringUtils.normalize(backgroundString).trim(); + backgroundString = normalize(backgroundString).trim(); } else { //Otherwise we basically want to treat the "Search" string and the display string //the same way. - backgroundString = StringUtils.normalize(raw.toString()); + backgroundString = normalize(raw.toString()); } - String normalizedDisplayString = StringUtils.normalize(raw.toString()); + String normalizedDisplayString = normalize(raw.toString()); removeSpans(raw); diff --git a/app/src/org/commcare/views/EntityViewTile.java b/app/src/org/commcare/views/EntityViewTile.java index 7ed36576c6..5160a8b0f1 100644 --- a/app/src/org/commcare/views/EntityViewTile.java +++ b/app/src/org/commcare/views/EntityViewTile.java @@ -1,15 +1,10 @@ package org.commcare.views; -import androidx.appcompat.app.AppCompatActivity; import android.content.Context; import android.content.res.Configuration; import android.graphics.Bitmap; import android.graphics.Color; import android.graphics.Point; -import android.os.Build; -import androidx.core.util.Pair; -import androidx.legacy.widget.Space; -import androidx.gridlayout.widget.GridLayout; import android.text.Spannable; import android.util.DisplayMetrics; import android.util.Log; @@ -20,9 +15,14 @@ import android.widget.ImageView.ScaleType; import android.widget.TextView; +import androidx.appcompat.app.AppCompatActivity; +import androidx.core.util.Pair; +import androidx.gridlayout.widget.GridLayout; +import androidx.legacy.widget.Space; + +import org.commcare.cases.entity.AsyncEntity; import org.commcare.cases.entity.Entity; import org.commcare.dalvik.R; -import org.commcare.models.AsyncEntity; import org.commcare.suite.model.Detail; import org.commcare.util.GridCoordinate; import org.commcare.util.GridStyle; diff --git a/app/src/org/commcare/xml/AndroidBulkCaseXmlParser.java b/app/src/org/commcare/xml/AndroidBulkCaseXmlParser.java index 488c449e22..d1fa69ce5c 100644 --- a/app/src/org/commcare/xml/AndroidBulkCaseXmlParser.java +++ b/app/src/org/commcare/xml/AndroidBulkCaseXmlParser.java @@ -7,7 +7,7 @@ import org.commcare.cases.model.Case; import org.commcare.models.database.SqlStorage; import org.commcare.models.database.user.models.AndroidCaseIndexTable; -import org.commcare.models.database.user.models.EntityStorageCache; +import org.commcare.models.database.user.models.CommCareEntityStorageCache; import org.commcare.xml.bulk.BulkProcessingCaseXmlParser; import org.javarosa.xml.util.InvalidStructureException; import org.kxml2.io.KXmlParser; @@ -26,21 +26,21 @@ * @author ctsims */ public class AndroidBulkCaseXmlParser extends BulkProcessingCaseXmlParser { - private final EntityStorageCache mEntityCache; + private final CommCareEntityStorageCache mEntityCache; private final AndroidCaseIndexTable mCaseIndexTable; private final SqlStorage storage; public AndroidBulkCaseXmlParser(KXmlParser parser, SqlStorage storage) { - this(parser, storage, new EntityStorageCache("case"), new AndroidCaseIndexTable()); + this(parser, storage, new CommCareEntityStorageCache("case"), new AndroidCaseIndexTable()); } public AndroidBulkCaseXmlParser(KXmlParser parser, SqlStorage storage, - EntityStorageCache entityStorageCache, + CommCareEntityStorageCache commCareEntityStorageCache, AndroidCaseIndexTable indexTable) { super(parser); - mEntityCache = entityStorageCache; + mEntityCache = commCareEntityStorageCache; mCaseIndexTable = indexTable; this.storage = storage; } diff --git a/app/src/org/commcare/xml/AndroidCaseXmlParser.java b/app/src/org/commcare/xml/AndroidCaseXmlParser.java index b7fb622cc6..5120566b07 100644 --- a/app/src/org/commcare/xml/AndroidCaseXmlParser.java +++ b/app/src/org/commcare/xml/AndroidCaseXmlParser.java @@ -12,7 +12,7 @@ import org.commcare.engine.references.JavaHttpReference; import org.commcare.interfaces.CommcareRequestEndpoints; import org.commcare.models.database.user.models.AndroidCaseIndexTable; -import org.commcare.models.database.user.models.EntityStorageCache; +import org.commcare.models.database.user.models.CommCareEntityStorageCache; import org.commcare.util.LogTypes; import org.commcare.utils.FileUtil; import org.commcare.utils.GlobalConstants; @@ -39,18 +39,18 @@ public class AndroidCaseXmlParser extends CaseXmlParser { private final boolean processAttachments = true; private CommcareRequestEndpoints generator; @Nullable - private final EntityStorageCache mEntityCache; + private final CommCareEntityStorageCache mEntityCache; private final AndroidCaseIndexTable mCaseIndexTable; public AndroidCaseXmlParser(KXmlParser parser, IStorageUtilityIndexed storage, - @Nullable EntityStorageCache entityCache, AndroidCaseIndexTable indexTable) { + @Nullable CommCareEntityStorageCache entityCache, AndroidCaseIndexTable indexTable) { super(parser, storage); mEntityCache = entityCache; mCaseIndexTable = indexTable; } public AndroidCaseXmlParser(KXmlParser parser, IStorageUtilityIndexed storage) { - this(parser, storage, new EntityStorageCache("case"), new AndroidCaseIndexTable()); + this(parser, storage, new CommCareEntityStorageCache("case"), new AndroidCaseIndexTable()); } public AndroidCaseXmlParser(KXmlParser parser, boolean acceptCreateOverwrites, @@ -58,7 +58,7 @@ public AndroidCaseXmlParser(KXmlParser parser, boolean acceptCreateOverwrites, CommcareRequestEndpoints generator) { super(parser, acceptCreateOverwrites, storage); this.generator = generator; - mEntityCache = new EntityStorageCache("case"); + mEntityCache = new CommCareEntityStorageCache("case"); mCaseIndexTable = new AndroidCaseIndexTable(); } diff --git a/app/unit-tests/src/org/commcare/android/util/TestUtils.java b/app/unit-tests/src/org/commcare/android/util/TestUtils.java index 86c04d4af6..56697c5f94 100644 --- a/app/unit-tests/src/org/commcare/android/util/TestUtils.java +++ b/app/unit-tests/src/org/commcare/android/util/TestUtils.java @@ -20,7 +20,7 @@ import org.commcare.models.database.SqlStorage; import org.commcare.models.database.user.DatabaseUserOpenHelper; import org.commcare.models.database.user.models.AndroidCaseIndexTable; -import org.commcare.models.database.user.models.EntityStorageCache; +import org.commcare.models.database.user.models.CommCareEntityStorageCache; import org.commcare.modern.database.TableBuilder; import org.commcare.test.utilities.CaseTestUtils; import org.commcare.utils.AndroidInstanceInitializer; @@ -48,7 +48,6 @@ import org.javarosa.xml.util.InvalidStructureException; import org.javarosa.xml.util.UnfullfilledRequirementsException; import org.javarosa.xpath.parser.XPathSyntaxException; -import org.robolectric.RuntimeEnvironment; import org.xmlpull.v1.XmlPullParserException; import java.io.IOException; @@ -113,14 +112,14 @@ private static TransactionParserFactory getFactory(final SQLiteDatabase db, fina //Note - this isn't even actually bulk processing. since this class is static //there's no good lifecycle to manage the bulk processor in, but at least //this will validate that the bulk processor works. - EntityStorageCache entityStorageCache = null; + CommCareEntityStorageCache commCareEntityStorageCache = null; if (CommCareApplication.instance().getCurrentApp() != null) { - entityStorageCache = new EntityStorageCache("case", db, AppUtils.getCurrentAppId()); + commCareEntityStorageCache = new CommCareEntityStorageCache("case", db, AppUtils.getCurrentAppId()); } if (bulkProcessingEnabled) { - return new AndroidBulkCaseXmlParser(parser, getCaseStorage(db), entityStorageCache, new AndroidCaseIndexTable(db)) { + return new AndroidBulkCaseXmlParser(parser, getCaseStorage(db), commCareEntityStorageCache, new AndroidCaseIndexTable(db)) { @Override protected SQLiteDatabase getDbHandle() { return db; @@ -128,7 +127,7 @@ protected SQLiteDatabase getDbHandle() { }; } else { - return new AndroidCaseXmlParser(parser, getCaseStorage(db), entityStorageCache, new AndroidCaseIndexTable(db)) { + return new AndroidCaseXmlParser(parser, getCaseStorage(db), commCareEntityStorageCache, new AndroidCaseIndexTable(db)) { @Override protected SQLiteDatabase getDbHandle() { return db;