Revision: 7743
http://datanucleus.svn.sourceforge.net/datanucleus/?rev=7743&view=rev
Author: andy_jefferson
Date: 2009-09-24 07:43:09 +0000 (Thu, 24 Sep 2009)
Log Message:
-----------
[NUCEXCEL-25] Use lazy loading of candidate instances. We still need to make use of weak referencing so that all instances aren't retained in memory. Also still need to provide better size() method handling that doesn't just load all instances
Modified Paths:
--------------
platform/store.excel/trunk/META-INF/MANIFEST.MF
platform/store.excel/trunk/pom.xml
platform/store.excel/trunk/src/java/org/datanucleus/store/excel/ExcelUtils.java
platform/store.excel/trunk/src/java/org/datanucleus/store/excel/query/JDOQLQuery.java
platform/store.excel/trunk/src/java/org/datanucleus/store/excel/query/JPQLQuery.java
Added Paths:
-----------
platform/store.excel/trunk/src/java/org/datanucleus/store/excel/query/ExcelCandidateList.java
Modified: platform/store.excel/trunk/META-INF/MANIFEST.MF
===================================================================
--- platform/store.excel/trunk/META-INF/MANIFEST.MF 2009-09-24 07:33:50 UTC (rev 7742)
+++ platform/store.excel/trunk/META-INF/MANIFEST.MF 2009-09-24 07:43:09 UTC (rev 7743)
@@ -2,9 +2,9 @@
Bundle-ManifestVersion: 2
Bundle-Name: Excel Plug-in
Bundle-SymbolicName: org.datanucleus.store.excel;singleton:=true
-Bundle-Version: 2.0.0.m2
+Bundle-Version: 2.0.0.m3
Bundle-Vendor: DataNucleus
-Require-Bundle: org.datanucleus;bundle-version="2.0.0.m1"
+Require-Bundle: org.datanucleus;bundle-version="2.0.0.m3"
Import-Package: org.apache.poi,
org.apache.poi.ddf,
org.apache.poi.hssf.eventmodel,
Modified: platform/store.excel/trunk/pom.xml
===================================================================
--- platform/store.excel/trunk/pom.xml 2009-09-24 07:33:50 UTC (rev 7742)
+++ platform/store.excel/trunk/pom.xml 2009-09-24 07:43:09 UTC (rev 7743)
@@ -2,7 +2,7 @@
<project>
<modelVersion>4.0.0</modelVersion>
<groupId>org.datanucleus</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.0-m3-SNAPSHOT</version>
<artifactId>datanucleus-excel</artifactId>
<name>DataNucleus EXCEL plugin</name>
<description>
@@ -81,7 +81,10 @@
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
- <configuration/>
+ <configuration>
+ <source>1.5</source>
+ <target>1.5</target>
+ </configuration>
</plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
Modified: platform/store.excel/trunk/src/java/org/datanucleus/store/excel/ExcelUtils.java
===================================================================
--- platform/store.excel/trunk/src/java/org/datanucleus/store/excel/ExcelUtils.java 2009-09-24 07:33:50 UTC (rev 7742)
+++ platform/store.excel/trunk/src/java/org/datanucleus/store/excel/ExcelUtils.java 2009-09-24 07:43:09 UTC (rev 7743)
@@ -17,16 +17,10 @@
**********************************************************************/
package org.datanucleus.store.excel;
-import java.util.ArrayList;
-import java.util.List;
-
import org.apache.poi.hssf.usermodel.HSSFCell;
import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
-import org.datanucleus.ClassLoaderResolver;
-import org.datanucleus.FetchPlan;
-import org.datanucleus.ObjectManager;
import org.datanucleus.StateManager;
import org.datanucleus.exceptions.NucleusDataStoreException;
import org.datanucleus.exceptions.NucleusException;
@@ -36,9 +30,6 @@
import org.datanucleus.metadata.IdentityMetaData;
import org.datanucleus.metadata.IdentityType;
import org.datanucleus.metadata.VersionMetaData;
-import org.datanucleus.store.FieldValues;
-import org.datanucleus.store.connection.ManagedConnection;
-import org.datanucleus.store.excel.fieldmanager.FetchFieldManager;
import org.datanucleus.util.Localiser;
import org.datanucleus.util.StringUtils;
@@ -430,103 +421,4 @@
return numRows;
}
-
- /**
- * Convenience method to get all objects of the candidate type (and optional subclasses) from the
- * specified workbook connection.
- * @param om ObjectManager
- * @param mconn Managed Connection
- * @param candidateClass Candidate
- * @param subclasses Include subclasses?
- * @param ignoreCache Whether to ignore the cache
- * @return List of objects of the candidate type
- */
- public static List getObjectsOfCandidateType(ObjectManager om, ManagedConnection mconn,
- Class candidateClass, boolean subclasses, boolean ignoreCache)
- {
- HSSFWorkbook wb = (HSSFWorkbook) mconn.getConnection();
- ClassLoaderResolver clr = om.getClassLoaderResolver();
- final AbstractClassMetaData acmd = om.getMetaDataManager().getMetaDataForClass(candidateClass, clr);
- List results = getObjectsOfCandidateType(om, wb, acmd, ignoreCache);
- if (subclasses)
- {
- // Add on any subclass objects
- String[] subclassNames = om.getMetaDataManager().getSubclassesForClass(candidateClass.getName(), true);
- if (subclassNames != null)
- {
- for (int i=0;i<subclassNames.length;i++)
- {
- AbstractClassMetaData cmd = om.getMetaDataManager().getMetaDataForClass(subclassNames[i], clr);
- results.addAll(getObjectsOfCandidateType(om, wb, cmd, ignoreCache));
- }
- }
- }
- return results;
- }
-
- /**
- * Method to extract the actual objects of a particular class from the Excel spreadsheet.
- * @param om ObjectManager
- * @param wb Workbook
- * @param acmd MetaData for the class
- * @param ignoreCache Whether to ignore the cache
- * @return List of objects (connected to StateManagers as required)
- */
- private static List getObjectsOfCandidateType(ObjectManager om, HSSFWorkbook wb,
- final AbstractClassMetaData acmd, boolean ignoreCache)
- {
- List results = new ArrayList();
-
- ClassLoaderResolver clr = om.getClassLoaderResolver();
- String sheetName = ExcelUtils.getSheetNameForClass(acmd);
- final HSSFSheet sheet = wb.getSheet(sheetName);
- if (sheet != null)
- {
- int idColIndex = -1;
- if (acmd.getIdentityType() == IdentityType.APPLICATION)
- {
- idColIndex = (int)ExcelUtils.getColumnIndexForFieldOfClass(acmd, acmd.getPKMemberPositions()[0]);
- }
- else if (acmd.getIdentityType() == IdentityType.DATASTORE)
- {
- idColIndex = (int)ExcelUtils.getColumnIndexForFieldOfClass(acmd, -1);
- }
- else
- {
- // Nondurable identity not supported so just return
- return results;
- }
-
- for (int i=0; i<sheet.getLastRowNum()+1 && sheet.getRow(i)!=null; i++)
- {
- final int row = i;
- final int col = 0;
- if (sheet.getRow(i).getCell(idColIndex) == null)
- {
- continue; // Inactive row
- }
-
- results.add(om.findObjectUsingAID(clr.classForName(acmd.getFullClassName()),
- new FieldValues()
- {
- // StateManager calls the fetchFields method
- public void fetchFields(StateManager sm)
- {
- sm.replaceFields(acmd.getAllMemberPositions(),
- new FetchFieldManager(sm,sheet, row, col));
- }
- public void fetchNonLoadedFields(StateManager sm)
- {
- sm.replaceNonLoadedFields(acmd.getAllMemberPositions(),
- new FetchFieldManager(sm,sheet, row, col));
- }
- public FetchPlan getFetchPlanForLoading()
- {
- return null;
- }
- }, ignoreCache, true));
- }
- }
- return results;
- }
}
\ No newline at end of file
Added: platform/store.excel/trunk/src/java/org/datanucleus/store/excel/query/ExcelCandidateList.java
===================================================================
--- platform/store.excel/trunk/src/java/org/datanucleus/store/excel/query/ExcelCandidateList.java (rev 0)
+++ platform/store.excel/trunk/src/java/org/datanucleus/store/excel/query/ExcelCandidateList.java 2009-09-24 07:43:09 UTC (rev 7743)
@@ -0,0 +1,166 @@
+/**********************************************************************
+Copyright (c) 2009 Andy Jefferson and others. All rights reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Contributors:
+ ...
+**********************************************************************/
+package org.datanucleus.store.excel.query;
+
+import java.util.NoSuchElementException;
+
+import org.apache.poi.hssf.usermodel.HSSFSheet;
+import org.apache.poi.hssf.usermodel.HSSFWorkbook;
+import org.datanucleus.ClassLoaderResolver;
+import org.datanucleus.FetchPlan;
+import org.datanucleus.ObjectManager;
+import org.datanucleus.StateManager;
+import org.datanucleus.metadata.AbstractClassMetaData;
+import org.datanucleus.metadata.IdentityType;
+import org.datanucleus.store.FieldValues;
+import org.datanucleus.store.connection.ManagedConnection;
+import org.datanucleus.store.excel.ExcelUtils;
+import org.datanucleus.store.excel.fieldmanager.FetchFieldManager;
+import org.datanucleus.store.query.AbstractCandidateList;
+
+/**
+ * Wrapper for a List of candidate instances from Excel. Loads the instances from the workbook lazily.
+ */
+public class ExcelCandidateList extends AbstractCandidateList
+{
+ /** Current worksheet, when we are reading from a particular worksheet. */
+ protected HSSFSheet worksheet;
+
+ /** Current row number in the current worksheet. */
+ protected int worksheetRowNum = -1;
+
+ public ExcelCandidateList(Class cls, boolean subclasses, ManagedConnection mconn, ObjectManager om,
+ boolean ignoreCache)
+ {
+ super(cls, subclasses, mconn, om, ignoreCache);
+ }
+
+ /**
+ * Method to get the next element in the List from the Excel spreadsheet.
+ * @return The object
+ * @throws NoSuchElementException if there is no "next" element
+ */
+ protected Object getNextObject()
+ {
+ if (cmds.isEmpty())
+ {
+ throw new NoSuchElementException();
+ }
+
+ AbstractClassMetaData cmd = cmds.iterator().next();
+ ClassLoaderResolver clr = om.getClassLoaderResolver();
+ if (worksheet != null)
+ {
+ // Take the next row in this worksheet
+ int idColIndex = -1;
+ if (cmd.getIdentityType() == IdentityType.APPLICATION)
+ {
+ idColIndex = (int)ExcelUtils.getColumnIndexForFieldOfClass(cmd, cmd.getPKMemberPositions()[0]);
+ }
+ else if (cmd.getIdentityType() == IdentityType.DATASTORE)
+ {
+ idColIndex = (int)ExcelUtils.getColumnIndexForFieldOfClass(cmd, -1);
+ }
+ else
+ {
+ // Nondurable not supported
+ throw new NoSuchElementException();
+ }
+
+ while (true)
+ {
+ if (worksheet.getRow(worksheetRowNum).getCell(idColIndex) == null)
+ {
+ // Inactive row, so ignore
+ worksheetRowNum++;
+ if (worksheetRowNum > worksheet.getLastRowNum())
+ {
+ break;
+ }
+ }
+ else
+ {
+ break;
+ }
+ }
+
+ if (worksheetRowNum <= worksheet.getLastRowNum())
+ {
+ final int row = worksheetRowNum;
+ final int col = 0;
+ final AbstractClassMetaData rowCmd = cmd;
+ Object obj = om.findObjectUsingAID(clr.classForName(cmd.getFullClassName()),
+ new FieldValues()
+ {
+ // StateManager calls the fetchFields method
+ public void fetchFields(StateManager sm)
+ {
+ sm.replaceFields(rowCmd.getAllMemberPositions(),
+ new FetchFieldManager(sm, worksheet, row, col));
+ }
+ public void fetchNonLoadedFields(StateManager sm)
+ {
+ sm.replaceNonLoadedFields(rowCmd.getAllMemberPositions(),
+ new FetchFieldManager(sm, worksheet, row, col));
+ }
+ public FetchPlan getFetchPlanForLoading()
+ {
+ return null;
+ }
+ }, ignoreCache, true);
+ worksheetRowNum++;
+
+ if (worksheetRowNum > worksheet.getLastRowNum())
+ {
+ // Finished this worksheet/class
+ cmds.remove(cmd);
+ worksheet = null;
+ worksheetRowNum = -1;
+ }
+ return obj;
+ }
+ else
+ {
+ // Reached end of worksheet with no more active rows, so go to next class if available
+ cmds.remove(cmd);
+ worksheet = null;
+ worksheetRowNum = -1;
+
+ // Repeat this method
+ return getNextObject();
+ }
+ }
+ else
+ {
+ // Starting class (no current worksheet)
+ String sheetName = ExcelUtils.getSheetNameForClass(cmd);
+ HSSFWorkbook workbook = (HSSFWorkbook) mconn.getConnection();
+ worksheet = workbook.getSheet(sheetName);
+ worksheetRowNum = 0;
+
+ if (worksheet != null)
+ {
+ return getNextObject();
+ }
+ else
+ {
+ throw new NoSuchElementException();
+ }
+ }
+ }
+}
\ No newline at end of file
Modified: platform/store.excel/trunk/src/java/org/datanucleus/store/excel/query/JDOQLQuery.java
===================================================================
--- platform/store.excel/trunk/src/java/org/datanucleus/store/excel/query/JDOQLQuery.java 2009-09-24 07:33:50 UTC (rev 7742)
+++ platform/store.excel/trunk/src/java/org/datanucleus/store/excel/query/JDOQLQuery.java 2009-09-24 07:43:09 UTC (rev 7743)
@@ -24,7 +24,6 @@
import java.util.Map;
import org.datanucleus.store.connection.ManagedConnection;
-import org.datanucleus.store.excel.ExcelUtils;
import org.datanucleus.ObjectManager;
import org.datanucleus.query.evaluator.JDOQLEvaluator;
import org.datanucleus.query.evaluator.JavaQueryEvaluator;
@@ -83,11 +82,11 @@
{
NucleusLogger.QUERY.debug(LOCALISER.msg("021046", "JDOQL", getSingleStringQuery(), null));
}
+
List candidates = null;
if (candidateCollection == null)
{
- candidates = ExcelUtils.getObjectsOfCandidateType(om, mconn, candidateClass, subclasses,
- ignoreCache);
+ candidates = new ExcelCandidateList(candidateClass, subclasses, mconn, om, ignoreCache);
}
else
{
Modified: platform/store.excel/trunk/src/java/org/datanucleus/store/excel/query/JPQLQuery.java
===================================================================
--- platform/store.excel/trunk/src/java/org/datanucleus/store/excel/query/JPQLQuery.java 2009-09-24 07:33:50 UTC (rev 7742)
+++ platform/store.excel/trunk/src/java/org/datanucleus/store/excel/query/JPQLQuery.java 2009-09-24 07:43:09 UTC (rev 7743)
@@ -24,7 +24,6 @@
import java.util.Map;
import org.datanucleus.store.connection.ManagedConnection;
-import org.datanucleus.store.excel.ExcelUtils;
import org.datanucleus.ObjectManager;
import org.datanucleus.exceptions.NucleusException;
import org.datanucleus.query.evaluator.JPQLEvaluator;
@@ -88,8 +87,7 @@
List candidates = null;
if (candidateCollection == null)
{
- candidates = ExcelUtils.getObjectsOfCandidateType(om, mconn, candidateClass, subclasses,
- ignoreCache);
+ candidates = new ExcelCandidateList(candidateClass, subclasses, mconn, om, ignoreCache);
}
else
{
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|