From: <ssk...@vh...> - 2005-12-09 13:23:58
|
Author: sskracic Date: 2005-12-09 14:21:16 +0100 (Fri, 09 Dec 2005) New Revision: 1042 Added: trunk/ccm-core/src/com/arsdigita/search/lucene/SqlFilter.java Removed: trunk/ccm-core/src/com/arsdigita/search/lucene/CategoryFilter.java Modified: trunk/ccm-cms/src/com/arsdigita/cms/Initializer.java trunk/ccm-cms/src/com/arsdigita/cms/search/LuceneQueryEngine.java trunk/ccm-cms/src/com/arsdigita/cms/ui/search/ItemQueryComponent.java trunk/ccm-core/pdl/com/arsdigita/search/Search.pdl trunk/ccm-core/src/com/arsdigita/search/lucene/BaseQueryEngine.java trunk/ccm-core/src/com/arsdigita/search/lucene/Initializer.java trunk/ccm-ldn-search/src/com/arsdigita/london/search/ui/AdvancedQueryComponent.java trunk/ccm-ldn-search/src/com/arsdigita/london/search/ui/SimpleQueryComponent.java Log: Implemented SqlFilter for Lucene search, which filters the search result against the arbitrary DataQuery. CategoryFilter has been obsoleted by this, so it's removed completely. Implemented PermissionFilter for Lucene using the SqlFilter and registered newly supported filter type in query engine registries. Finally, enabled new components in UI. Modified: trunk/ccm-cms/src/com/arsdigita/cms/Initializer.java =================================================================== --- trunk/ccm-cms/src/com/arsdigita/cms/Initializer.java 2005-12-09 13:10:03 UTC (rev 1041) +++ trunk/ccm-cms/src/com/arsdigita/cms/Initializer.java 2005-12-09 13:21:16 UTC (rev 1042) @@ -251,6 +251,7 @@ new LastModifiedDateFilterType(), new LastModifiedUserFilterType(), new ObjectTypeFilterType(), + new PermissionFilterType(), new VersionFilterType() }, new LuceneQueryEngine()); Modified: trunk/ccm-cms/src/com/arsdigita/cms/search/LuceneQueryEngine.java =================================================================== --- trunk/ccm-cms/src/com/arsdigita/cms/search/LuceneQueryEngine.java 2005-12-09 13:10:03 UTC (rev 1041) +++ trunk/ccm-cms/src/com/arsdigita/cms/search/LuceneQueryEngine.java 2005-12-09 13:21:16 UTC (rev 1042) @@ -18,23 +18,19 @@ */ package com.arsdigita.cms.search; -import com.arsdigita.categorization.Category; import com.arsdigita.cms.ContentType; import com.arsdigita.cms.search.ContentTypeFilterSpecification; import com.arsdigita.cms.search.ContentTypeFilterType; import com.arsdigita.kernel.PartyCollection; -import com.arsdigita.persistence.DataQuery; -import com.arsdigita.persistence.SessionManager; import com.arsdigita.search.FilterSpecification; import com.arsdigita.search.FilterType; -import com.arsdigita.search.filters.CategoryFilterSpecification; import com.arsdigita.search.filters.DateRangeFilterSpecification; import com.arsdigita.search.filters.PartyFilterSpecification; import com.arsdigita.search.lucene.BaseQueryEngine; -import com.arsdigita.search.lucene.CategoryFilter; import com.arsdigita.search.lucene.Document; import com.arsdigita.search.lucene.ObjectTypeFilter; import com.arsdigita.search.lucene.PartyFilter; +import com.arsdigita.search.lucene.SqlFilter; import com.arsdigita.search.lucene.TypeSpecificFilter; import com.arsdigita.search.lucene.UnionFilter; import java.util.ArrayList; @@ -46,9 +42,9 @@ public class LuceneQueryEngine extends BaseQueryEngine { - protected void addFilter(List list, + protected void addFilter(List list, SqlFilter sql, FilterSpecification filter) { - super.addFilter(list, filter); + super.addFilter(list, sql, filter); FilterType type = filter.getType(); @@ -124,23 +120,8 @@ list.add(new UnionFilter(filters)); } - protected void addCategoryFilter(List list, - CategoryFilterSpecification filterSpec) { - Category[] cats = filterSpec.getCategories(); - if (cats == null || cats.length == 0) { - return; - } - list.add(new CategoryFilter(cats, filterSpec.isDescending()) { - protected DataQuery getQuery(List catList, boolean descending) { - DataQuery dq = SessionManager.getSession() - .retrieveQuery("com.arsdigita.cms.searchCategoryObjects"); - dq.setParameter("ids", catList); - // 999999 - just need a number that is grater than max. category - // tree depth - dq.setParameter("pathLimit", new Integer(descending ? 999999 : 0)); - return dq; - } - }); - + protected String getCategorisationQuery() { + return "com.arsdigita.cms.searchCategoryObjects"; } + } Modified: trunk/ccm-cms/src/com/arsdigita/cms/ui/search/ItemQueryComponent.java =================================================================== --- trunk/ccm-cms/src/com/arsdigita/cms/ui/search/ItemQueryComponent.java 2005-12-09 13:10:03 UTC (rev 1041) +++ trunk/ccm-cms/src/com/arsdigita/cms/ui/search/ItemQueryComponent.java 2005-12-09 13:21:16 UTC (rev 1042) @@ -56,8 +56,6 @@ m_context = context; if (Search.getConfig().isIntermediaEnabled()) { - add(new PermissionFilterComponent( - SecurityManager.CMS_READ_ITEM)); add(new LaunchDateFilterWidget(new LaunchDateFilterType(), LaunchDateFilterType.KEY)); } @@ -65,6 +63,9 @@ if (Search.getConfig().isIntermediaEnabled() || Search.getConfig().isLuceneEnabled()) { + add(new PermissionFilterComponent( + SecurityManager.CMS_READ_ITEM)); + add(new SimpleCategoryFilterWidget() { protected Category[] getRoots(PageState state) { Category[] roots; Modified: trunk/ccm-core/pdl/com/arsdigita/search/Search.pdl =================================================================== --- trunk/ccm-core/pdl/com/arsdigita/search/Search.pdl 2005-12-09 13:10:03 UTC (rev 1041) +++ trunk/ccm-core/pdl/com/arsdigita/search/Search.pdl 2005-12-09 13:21:16 UTC (rev 1042) @@ -35,3 +35,32 @@ id = m.object_id; } } + +query getLuceneDocIDs { + BigDecimal id; + + do { + select ld.document_id + from lucene_docs ld + where ld.is_deleted = '0' + } map { + id = ld.document_id; + } +} + +query partyPermissionFilterStub { + BigDecimal id; + do { + select dogc.pd_object_id + from dnm_object_1_granted_context dogc, + dnm_granted_context dgc, + dnm_permissions dp, + dnm_group_membership dgm + where dogc.pd_context_id = dgc.pd_object_id + and dgc.pd_context_id = dp.pd_object_id + and dgm.pd_member_id in :partyID + and dp.pd_grantee_id = dgm.pd_group_id + } map { + id = dogc.pd_object_id; + } +} Modified: trunk/ccm-core/src/com/arsdigita/search/lucene/BaseQueryEngine.java =================================================================== --- trunk/ccm-core/src/com/arsdigita/search/lucene/BaseQueryEngine.java 2005-12-09 13:10:03 UTC (rev 1041) +++ trunk/ccm-core/src/com/arsdigita/search/lucene/BaseQueryEngine.java 2005-12-09 13:21:16 UTC (rev 1042) @@ -19,6 +19,11 @@ package com.arsdigita.search.lucene; import com.arsdigita.categorization.Category; +import com.arsdigita.kernel.permissions.PermissionManager; +import com.arsdigita.kernel.permissions.PermissionService; +import com.arsdigita.kernel.permissions.PrivilegeDescriptor; +import com.arsdigita.kernel.permissions.UniversalPermissionDescriptor; +import com.arsdigita.persistence.OID; import com.arsdigita.persistence.metadata.ObjectType; import com.arsdigita.search.FilterSpecification; import com.arsdigita.search.FilterType; @@ -32,6 +37,8 @@ import com.arsdigita.search.filters.ContentSectionFilterType; import com.arsdigita.search.filters.ObjectTypeFilterSpecification; import com.arsdigita.search.filters.ObjectTypeFilterType; +import com.arsdigita.search.filters.PermissionFilterSpecification; +import com.arsdigita.search.filters.PermissionFilterType; import java.util.ArrayList; import java.util.List; import org.apache.log4j.Logger; @@ -79,12 +86,16 @@ protected void addFilters(List list, FilterSpecification[] filters) { + SqlFilter sql = new SqlFilter(); for (int i = 0 ; i < filters.length ; i++) { - addFilter(list, filters[i]); + addFilter(list, sql, filters[i]); } + if (sql.getNumClauses() > 0) { + list.add(sql); + } } - protected void addFilter(List list, + protected void addFilter(List list, SqlFilter sql, FilterSpecification filter) { FilterType type = filter.getType(); @@ -93,19 +104,48 @@ } else if (ContentSectionFilterType.KEY.equals(type.getKey())) { addContentSectionFilter(list, (ContentSectionFilterSpecification)filter); } else if (CategoryFilterType.KEY.equals(type.getKey())) { - addCategoryFilter(list, (CategoryFilterSpecification)filter); + addCategoryFilter(sql, (CategoryFilterSpecification) filter); + } else if (PermissionFilterType.KEY.equals(type.getKey())) { + addPermissionFilter(sql, (PermissionFilterSpecification) filter); } } - protected void addCategoryFilter(List list, + protected String getCategorisationQuery() { + return "com.arsdigita.search.categoryObjects"; + } + + protected void addCategoryFilter(SqlFilter sql, CategoryFilterSpecification filterSpec) { Category[] cats = filterSpec.getCategories(); if (cats == null || cats.length == 0) { return; } - list.add(new CategoryFilter(cats)); + com.arsdigita.persistence.Filter f = sql.appendClause(getCategorisationQuery(), "id"); + List catList = new ArrayList(); + for (int i=0; i<cats.length; i++) { + catList.add(cats[i].getID()); + } + f.set("ids", catList); + // 999999 - just need a number that is grater than max. category + // tree depth + f.set("pathLimit", new Integer(filterSpec.isDescending() ? 999999 : 0)); } + protected void addPermissionFilter(SqlFilter sql, + PermissionFilterSpecification filterSpec) { + PrivilegeDescriptor privilege = filterSpec.getPrivilege(); + OID partyOID = filterSpec.getParty().getOID(); + UniversalPermissionDescriptor universalPermission = + new UniversalPermissionDescriptor(privilege, partyOID); + if (PermissionService.checkPermission(universalPermission)) { + return; + } + com.arsdigita.persistence.Filter f = sql.getFilterFactory().simple( + Document.ID + " in ( com.arsdigita.search.partyPermissionFilterStub " + + " and RAW[" + privilege.getColumnName() + " = 1 ])"); + f.set("partyID", PermissionManager.constructAccessList(partyOID)); + sql.appendClause(f); + } protected void addObjectTypeFilter(List list, ObjectTypeFilterSpecification filter) { @@ -147,7 +187,6 @@ // list.add(new UnionFilter(filters)); // } - - } } + Deleted: trunk/ccm-core/src/com/arsdigita/search/lucene/CategoryFilter.java Modified: trunk/ccm-core/src/com/arsdigita/search/lucene/Initializer.java =================================================================== --- trunk/ccm-core/src/com/arsdigita/search/lucene/Initializer.java 2005-12-09 13:10:03 UTC (rev 1041) +++ trunk/ccm-core/src/com/arsdigita/search/lucene/Initializer.java 2005-12-09 13:21:16 UTC (rev 1042) @@ -26,6 +26,7 @@ import com.arsdigita.search.Search; import com.arsdigita.search.filters.CategoryFilterType; import com.arsdigita.search.filters.ObjectTypeFilterType; +import com.arsdigita.search.filters.PermissionFilterType; import java.io.File; import java.io.IOException; import java.util.Date; @@ -100,7 +101,8 @@ QueryEngineRegistry.registerEngine (IndexerType.LUCENE, new FilterType[] { new CategoryFilterType(), - new ObjectTypeFilterType() + new ObjectTypeFilterType(), + new PermissionFilterType() }, new BaseQueryEngine()); } Added: trunk/ccm-core/src/com/arsdigita/search/lucene/SqlFilter.java =================================================================== --- trunk/ccm-core/src/com/arsdigita/search/lucene/SqlFilter.java 2005-12-09 13:10:03 UTC (rev 1041) +++ trunk/ccm-core/src/com/arsdigita/search/lucene/SqlFilter.java 2005-12-09 13:21:16 UTC (rev 1042) @@ -0,0 +1,137 @@ +/* + * Copyright (C) 2005 Red Hat Inc. All Rights Reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation; either version 2.1 of + * the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ +package com.arsdigita.search.lucene; + +import com.arsdigita.persistence.DataQuery; +import com.arsdigita.persistence.FilterFactory; +import com.arsdigita.persistence.SessionManager; +import com.arsdigita.search.lucene.Document; +import com.redhat.persistence.engine.rdbms.RDBMSEngine; +import java.io.IOException; +import java.util.BitSet; +import java.util.HashMap; +import java.util.Map; +import org.apache.log4j.Logger; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermDocs; +import org.apache.lucene.index.TermEnum; +import org.apache.lucene.search.Filter; + +/** + * + * A filter based on the arbitrary SQL query that returns + * document IDs that are allowed in search results. + * + * @author Sebastian Skracic (ssk...@re...) + * @version $Id$ + * + **/ + +public class SqlFilter extends Filter { + + private static final Logger s_log = Logger.getLogger(SqlFilter.class); + + private DataQuery m_query; + + private int m_clauses; + + public SqlFilter() { + m_query = SessionManager.getSession().retrieveQuery("com.arsdigita.search.getLuceneDocIDs"); + m_clauses = 0; + } + + /** + * Appends a clause to the SqlFilter + * + * @param query PDL query that will be used to constraint the result set + * @param propertyName name of the query property that holds the document ID column + * + **/ + public com.arsdigita.persistence.Filter appendClause(String query, String propertyName) { + m_clauses++; + return m_query.addInSubqueryFilter("id", propertyName, query); + } + + /** + * Appends a (persistence) filter to the SqlFilter + * + * @param f PDL filter to be ANDed with all other clauses + * + **/ + public void appendClause(com.arsdigita.persistence.Filter f) { + m_clauses++; + m_query.addFilter(f); + } + + /** + * Returns a BitSet with true for documents which + * should be permitted in search results, and false + * for those that should not. + **/ + final public BitSet bits(IndexReader reader) throws IOException { + + // First build CCM ID <-> lucene ID mappings + long st = System.currentTimeMillis(); + Map luceneIds = new HashMap(); + TermDocs docs = reader.termDocs(); + TermEnum terms = reader.terms(new Term(Document.ID, "")); + while (terms.next()) { + Term t = terms.term(); + String field = t.field(); + String text = t.text(); + if (!Document.ID.equals(field)) { + break; + } + docs.seek(t); + if (docs.next()) { + luceneIds.put(text, new Integer(docs.doc())); + } + } + terms.close(); + docs.close(); + s_log.debug("Mapped " + luceneIds.size() + " IDs in " + (System.currentTimeMillis() - st) + "ms"); + st = System.currentTimeMillis(); + BitSet bits = new BitSet(reader.maxDoc()); + s_log.debug("Created BitSet with " + bits.size() + " bits"); + int i=0; + + while (m_query.next()) { + String ccmID = m_query.get("id").toString(); + Integer luceneID = (Integer) luceneIds.get(ccmID); + if (luceneID == null) { + s_log.info("Could not find document ID " + ccmID + " in Lucene index"); + } else { + i++; + bits.set(luceneID.intValue()); + } + } + s_log.debug("Created SQLFilter with " + m_clauses + " clause(s) matching " + i + " object(s)"); + s_log.debug("Time taken: " + (System.currentTimeMillis() - st) + "ms"); + return bits; + } + + public int getNumClauses() { + return m_clauses; + } + + public FilterFactory getFilterFactory() { + return m_query.getFilterFactory(); + } +} Property changes on: trunk/ccm-core/src/com/arsdigita/search/lucene/SqlFilter.java ___________________________________________________________________ Name: svn:keywords + Id Author URL Modified: trunk/ccm-ldn-search/src/com/arsdigita/london/search/ui/AdvancedQueryComponent.java =================================================================== --- trunk/ccm-ldn-search/src/com/arsdigita/london/search/ui/AdvancedQueryComponent.java 2005-12-09 13:10:03 UTC (rev 1041) +++ trunk/ccm-ldn-search/src/com/arsdigita/london/search/ui/AdvancedQueryComponent.java 2005-12-09 13:21:16 UTC (rev 1042) @@ -54,15 +54,12 @@ public AdvancedQueryComponent(String context) { - if (Search.getConfig().isIntermediaEnabled()) { - add(new PermissionFilterComponent( - PrivilegeDescriptor.READ)); - - } - if (Search.getConfig().isIntermediaEnabled() || Search.getConfig().isLuceneEnabled()) { + add(new PermissionFilterComponent( + PrivilegeDescriptor.READ)); + Application app = Web.getContext().getApplication(); Category root = Category.getRootForObject(app); add(new SimpleCategoryFilterWidget(root)); Modified: trunk/ccm-ldn-search/src/com/arsdigita/london/search/ui/SimpleQueryComponent.java =================================================================== --- trunk/ccm-ldn-search/src/com/arsdigita/london/search/ui/SimpleQueryComponent.java 2005-12-09 13:10:03 UTC (rev 1041) +++ trunk/ccm-ldn-search/src/com/arsdigita/london/search/ui/SimpleQueryComponent.java 2005-12-09 13:21:16 UTC (rev 1042) @@ -5,12 +5,12 @@ * modify it under the terms of the GNU Lesser General Public License * as published by the Free Software Foundation; either version 2.1 of * the License, or (at your option) any later version. - * + * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. - * + * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA @@ -52,17 +52,13 @@ private Form m_form; public SimpleQueryComponent(String context) { - - // XXX READ instead of CMS_READ_ITEM - if (Search.getConfig().isIntermediaEnabled()) { - add(new PermissionFilterComponent(PrivilegeDescriptor.READ)); - } - + if (Search.getConfig().isIntermediaEnabled() || Search.getConfig().isLuceneEnabled()) { + add(new PermissionFilterComponent(PrivilegeDescriptor.READ)); add(new VersionFilterComponent(context)); } - + SearchConfig config = com.arsdigita.london.search.Search.getConfig(); s_log.debug("Array length : "+config.getSimpleRestrictToArray().length); if (config.getSimpleRestrictToArray().length > 0) { @@ -74,73 +70,73 @@ public void register(Form form, FormModel model) { - s_log.debug("Adding " + m_hiddenAllowedContentSectionsList.getName() + " to form model"); - m_hiddenAllowedContentSectionsList.setPassIn(true); - model.addFormParam(m_hiddenAllowedContentSectionsList); - super.register(form, model); - m_form = form; + s_log.debug("Adding " + m_hiddenAllowedContentSectionsList.getName() + " to form model"); + m_hiddenAllowedContentSectionsList.setPassIn(true); + model.addFormParam(m_hiddenAllowedContentSectionsList); + super.register(form, model); + m_form = form; } - + /** * Gets the hidden restrictToContentSections param. * The param is in the form of a comma seperated list of content section names. * If present the search will only return content items from these content sections. **/ protected String getContentSections(PageState state) { - FormData formData = m_form.getFormData(state); - if (formData != null) { - ParameterData contentSectionListParam = formData.getParameter(m_hiddenAllowedContentSectionsList.getName()); - String paramValue = (String)contentSectionListParam.getValue(); - m_paramValue = paramValue; - s_log.debug("content sections list is " + paramValue); - is_restricted = (m_paramValue != null && !"".equals(m_paramValue)); - return (String)contentSectionListParam.getValue(); - } - return null; + FormData formData = m_form.getFormData(state); + if (formData != null) { + ParameterData contentSectionListParam = formData.getParameter(m_hiddenAllowedContentSectionsList.getName()); + String paramValue = (String)contentSectionListParam.getValue(); + m_paramValue = paramValue; + s_log.debug("content sections list is " + paramValue); + is_restricted = (m_paramValue != null && !"".equals(m_paramValue)); + return (String)contentSectionListParam.getValue(); + } + return null; } /** * Gets an array of content section titles for creation of an inclusion filter. **/ public String[] getContentSectionsArray(PageState state) { - String contentSections = getContentSections(state); - if (contentSections == null) { return null; } - StringTokenizer st = new StringTokenizer(getContentSections(state), ","); - contentSectionTitles = new String[st.countTokens()]; - int index = 0; + String contentSections = getContentSections(state); + if (contentSections == null) { return null; } + StringTokenizer st = new StringTokenizer(getContentSections(state), ","); + contentSectionTitles = new String[st.countTokens()]; + int index = 0; while (st.hasMoreTokens()) { contentSectionTitles[index] = st.nextToken().trim(); s_log.info("Restricting to content section : "+contentSectionTitles[index]); index++; } - return contentSectionTitles; + return contentSectionTitles; } /** * Adds the content section filter to any existing filters. **/ protected FilterSpecification[] getFilters(PageState state) { - FilterSpecification[] existingfilters = super.getFilters(state); - List n = new ArrayList(); - try { - List filters = Arrays.asList(existingfilters); // this will throw a NullPointerException if there are no existing filters - n.addAll(filters); - } catch (NullPointerException e) { - // do we need to catch it if we're doing nothing with it? - } - String[] contentSections = getContentSectionsArray(state); - if (contentSections == null) { return existingfilters; } - ContentSectionFilterSpecification csfs = new ContentSectionFilterSpecification(contentSections); - n.add(csfs); - FilterSpecification[] newFilters = new FilterSpecification[n.size()]; - Iterator i = n.iterator(); - int c = 0; - while (i.hasNext()) { - newFilters[c] = (FilterSpecification)i.next(); - c++; - } + FilterSpecification[] existingfilters = super.getFilters(state); + List n = new ArrayList(); + try { + List filters = Arrays.asList(existingfilters); // this will throw a NullPointerException if there are no existing filters + n.addAll(filters); + } catch (NullPointerException e) { + // do we need to catch it if we're doing nothing with it? + } + String[] contentSections = getContentSectionsArray(state); + if (contentSections == null) { return existingfilters; } + ContentSectionFilterSpecification csfs = new ContentSectionFilterSpecification(contentSections); + n.add(csfs); + FilterSpecification[] newFilters = new FilterSpecification[n.size()]; + Iterator i = n.iterator(); + int c = 0; + while (i.hasNext()) { + newFilters[c] = (FilterSpecification)i.next(); + c++; + } - return newFilters; + return newFilters; } } |