Update of /var/cvs/contributions/CMSContainer_Modules/dataconversion/src/java/com/finalist/cmsc/dataconversion/service In directory james.mmbase.org:/tmp/cvs-serv19613/service
Modified Files: Constants.java Conversion.java Data.java NodeService.java XMLUtil.java Log Message: CMSC-1166, KNMI migration to the CMS Container,make some improvements to get the data imported properly See also: http://cvs.mmbase.org/viewcvs/contributions/CMSContainer_Modules/dataconversion/src/java/com/finalist/cmsc/dataconversion/service See also: http://www.mmbase.org/jira/browse/CMSC-1166 Index: Constants.java =================================================================== RCS file: /var/cvs/contributions/CMSContainer_Modules/dataconversion/src/java/com/finalist/cmsc/dataconversion/service/Constants.java,v retrieving revision 1.1 retrieving revision 1.2 diff -u -b -r1.1 -r1.2 --- Constants.java 6 May 2008 11:05:49 -0000 1.1 +++ Constants.java 28 Nov 2008 07:30:44 -0000 1.2 @@ -10,4 +10,6 @@ public static final Byte ROOT_CATEGORY_TYPE = 3; + public static final Byte RELATION_DATA_TYPE = 4; + } Index: Conversion.java =================================================================== RCS file: /var/cvs/contributions/CMSContainer_Modules/dataconversion/src/java/com/finalist/cmsc/dataconversion/service/Conversion.java,v retrieving revision 1.4 retrieving revision 1.5 diff -u -b -r1.4 -r1.5 --- Conversion.java 7 Oct 2008 13:05:59 -0000 1.4 +++ Conversion.java 28 Nov 2008 07:30:44 -0000 1.5 @@ -4,6 +4,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Properties; import javax.servlet.ServletContext; @@ -16,6 +17,7 @@ import org.w3c.dom.NodeList; import com.finalist.cmsc.dataconversion.dataaccess.DataAccessDelegate; +import com.finalist.cmsc.dataconversion.dataaccess.DataAccessor; import com.finalist.cmsc.dataconversion.dataaccess.DataHolder; import com.finalist.cmsc.dataconversion.dataaccess.DataSourceFactory; @@ -60,125 +62,199 @@ Document dom = null; long beginTime = System.currentTimeMillis(); try { - - log.info(" ############################### begin import ######################"); dom = XMLParser.parseXMLToDOM(context.getResourceAsStream(IMPORT_FILE)); // XMLParser.DTDValidator(dom); - Element element = (Element)dom.getElementsByTagName("import").item(0); + Element element = (Element) dom.getElementsByTagName("import").item(0); NodeList list = element.getChildNodes(); List<Data> sources = new ArrayList<Data>(); - for(int i = 0 ; i < list.getLength(); i++) { - //datatype node - if(list.item(i).getNodeName().equals("datatype")) { - //deal with data type node - log.info(String.format(table_log,XMLUtil.getSourceType((Element)list.item(i)))); - Data sorData = getDataOfDataType((Element)list.item(i)); - //load data number from - HashMap<Integer,Integer> keys = DataAccessDelegate.getNumbersOfDataType((Element)list.item(i), dataSource); - log.info(String.format(result_count,keys.size(),XMLUtil.getSourceType((Element)list.item(i)))); + List<Data> clondSources = new ArrayList<Data>(); + int j= list.getLength(); + for (int i = 0; i < j; i++) { + // datatype node + if (list.item(i).getNodeName().equals("datatype")) { + // deal with data type node + log.info(String.format(table_log, XMLUtil.getSourceType((Element) list.item(i)))); + Data sorData = getDataOfDataType((Element) list.item(i)); + // load data number from + HashMap<Integer, Integer> keys = DataAccessDelegate.getNumbersOfDataType((Element) list.item(i), + dataSource); + log.info(String.format(result_count, keys.size(), XMLUtil + .getSourceType((Element) list.item(i)))); sorData.setIdentifiers(keys); sources.add(sorData); - - if(XMLUtil.hasSelfRelation((Element)list.item(i))) { - log.info(String.format(sefRelation_log,XMLUtil.getSourceType((Element)list.item(i)))); - Data relData = getDataOfSelfRelation((Element)list.item(i)); - HashMap<Integer,Integer> keys1 = DataAccessDelegate.getNumbersOfSelfRelation((Element)list.item(i), dataSource); - log.info(String.format(result_count,keys1.size(),XMLUtil.getSourceType((Element)list.item(i)))); + //log.info("did I chage? "+Utils.getXML(list.item(i))); + if (XMLUtil.hasSelfRelation((Element) list.item(i))) { + log.info(String.format(sefRelation_log, XMLUtil.getSourceType((Element) list.item(i)))); + Data relData = getDataOfSelfRelation((Element) list.item(i)); + HashMap<Integer, Integer> keys1 = DataAccessDelegate.getNumbersOfSelfRelation((Element) list.item(i), + dataSource); + log.info(String.format(result_count, keys1.size(), XMLUtil.getSourceType((Element) list + .item(i)))); if (keys1 == null || keys1.isEmpty()) { continue; } relData.setIdentifiers(keys1); sources.add(relData); } - Data rootData = getRootData((Element)list.item(i)); - HashMap<Integer,Integer> keysd = DataAccessDelegate.addChildRelation((Element)list.item(i), dataSource); - log.info(String.format(sefRelation_log,keysd.size())); + Data rootData = getRootData((Element) list.item(i)); + HashMap<Integer, Integer> keysd = DataAccessDelegate + .addChildRelation((Element) list.item(i), dataSource); + log.info(String.format(sefRelation_log, keysd.size())); rootData.setIdentifiers(keysd); rootData.setRelateId(Integer.parseInt(node)); - - // sources.add(rootData); log.info("----> before recur"); - recur((Element)list.item(i),sources); + recur((Element) list.item(i), sources); log.info("----> after recur"); - for(Data data:sources) { - if(data.getType() == Constants.ENTITY_TYPE) { - log.info(String.format(" ----> begin to save type [%s]",data.getDestinationType())); + relDatatype((Element) list.item(i), sources); + saveType(list, sources, i); + log.info("entity type and it relation finished........."); + NodeService.insertData(null, rootData, sources, null); + } + //end if + clondSources.addAll(sources); + sources.clear(); + } + NodeService.insertMigrationMappings(clondSources); + createRelationDataType(clondSources); + linkRoot(list, j); + log.info(String.format( + " ---->#######################finished importing [time:%s mins] #############################", + (System.currentTimeMillis() - beginTime) / (1000 * 60))); + NodeService.insertProperties(properties); + } catch (Exception e) { + log.error(e.getMessage()); + e.printStackTrace(); } - else if(data.getType() == Constants.RELATION_TYPE) { - log.info(String.format(" ----> begin to save type [%s]",data.getDestinationRelationType())); } - else if(data.getType() == Constants.SELF_RELATION_TYPE) { - log.info(String.format(" ----> begin to save type [%s]",data.getSourceRelationType())); + + private void linkRoot(NodeList list, int j) { + Map<String, ArrayList<Integer>> rootDatas=new HashMap<String, ArrayList<Integer>>(); + for (int i = 0; i < j; i++) { + if (list.item(i).getNodeName().equals("datatype")) { + String destinationtype=XMLUtil.getDestinationType((Element) list.item(i)); + String sourcetype=XMLUtil.getSourceType((Element) list.item(i)); + DataAccessor da=new DataAccessor(dataSource); + ArrayList<Integer> primerKeys=da.getPrimerKeyList(sourcetype); + rootDatas.put(destinationtype, primerKeys); + } + } + NodeService.linkRootDatas(rootDatas,node); + } + + private void createRelationDataType(List<Data> sources) throws Exception { + List<String> reskeys = new ArrayList<String>(); + for (Data reldata : sources) { + if (reldata.getType() == Constants.RELATION_DATA_TYPE) { + reskeys = DataAccessDelegate.getResOfRelation(sources, reldata, dataSource); + NodeService.createRelationData(reskeys, reldata); + } + } + } + + private void saveType(NodeList list, List<Data> sources, int i) throws Exception { + for (Data data : sources) { + if (data.getType() != Constants.RELATION_DATA_TYPE) { + if (data.getType() == Constants.ENTITY_TYPE) { + log.info(String.format(" ----> begin to save type [%s]", data.getDestinationType())); + } else if (data.getType() == Constants.RELATION_TYPE) { + log.info(String.format(" ----> begin to save type [%s]", data.getDestinationRelationType())); + } else if (data.getType() == Constants.SELF_RELATION_TYPE) { + log.info(String.format(" ----> begin to save type [%s]", data.getSourceRelationType())); } Iterator<Integer> iterator = data.getIdentifiers().keySet().iterator(); - while(iterator.hasNext()) { + while (iterator.hasNext()) { Integer key = iterator.next(); - Element elment = XMLUtil.getElementByTableName(data.getTableName(),(Element)list.item(i)); - DataHolder holder = DataAccessDelegate.getElementByPrimaryKey(elment, dataSource, key,data.getType()); - Integer number = NodeService.insertData(holder,data,sources,key); + Element elment = XMLUtil.getElementByTableName(data.getTableName(), (Element) list.item(i)); + DataHolder holder = DataAccessDelegate.getElementByPrimaryKey(elment, dataSource, key, data.getType()); + Integer number = NodeService.insertData(holder, data, sources, key); + if (null!=number) { data.getIdentifiers().put(key, number); } - log.info("----> end saving "); - } - System.out.println("finished........."); - NodeService.insertData(null,rootData,sources,null); - } } - log.info(String.format(" ---->#######################finished importing [time:%s mins] #############################",(System.currentTimeMillis()-beginTime)/(1000*60))); - NodeService.insertProperties(properties); + log.info("----> end saving "); } - catch (Exception e) { - log.error(e.getMessage()); - e.printStackTrace(); } } + /** * recur to get elements related - * @param element dom element - * @param sources a collection object which hold Data Objects + * + * @param element + * dom element + * @param sources + * a collection object which hold Data Objects * @throws Exception */ - private void recur(Element element,List<Data> sources) throws Exception { + private void recur(Element element, List<Data> sources) throws Exception { Element[] relates = XMLUtil.getDirectRelateChildNodes(element); - if(relates.length < 1) { - return ; + if (relates.length < 1) { + return; } - for(int i = 0 ; i < relates.length ; i++) { - log.info("###################### begin Import table ["+XMLUtil.getSourceType(relates[i])+"] ####################"); - log.info(String.format(table_log,XMLUtil.getSourceType(relates[i]))); + for (int i = 0; i < relates.length; i++) { + log.info("###################### begin Import table [" + XMLUtil.getSourceType(relates[i]) + + "] ####################"); + log.info(String.format(table_log, XMLUtil.getSourceType(relates[i]))); Data sorData = getDataOfDataType(relates[i]); - HashMap<Integer,Integer> keys = DataAccessDelegate.getNumbersOfDataType(relates[i], dataSource); - log.info(String.format(result_count,keys.size(),XMLUtil.getSourceType(relates[i]))); + HashMap<Integer, Integer> keys = DataAccessDelegate.getNumbersOfDataType(relates[i], dataSource); + log.info(String.format(result_count, keys.size(), XMLUtil.getSourceType(relates[i]))); if (keys == null || keys.isEmpty()) { continue; } sorData.setIdentifiers(keys); sources.add(sorData); - log.info(String.format(relation_log,XMLUtil.getSourceType(element),XMLUtil.getSourceType(relates[i]),XMLUtil.getSourceRelationType(relates[i]))); + log.info(String.format(relation_log, XMLUtil.getSourceType(element), XMLUtil + .getSourceType(relates[i]), XMLUtil.getSourceRelationType(relates[i]))); Data relData = getDataOfRelation(relates[i]); - HashMap<Integer,Integer> keys1 = DataAccessDelegate.getNumbersOfRelation(relates[i], dataSource); - log.info(String.format(result_rel_count,keys1.size(),XMLUtil.getSourceRelationType(relates[i]))); + HashMap<Integer, Integer> keys1 = DataAccessDelegate.getNumbersOfRelation(relates[i], dataSource); + log.info(String.format(result_rel_count, keys1.size(), XMLUtil.getSourceRelationType(relates[i]))); if (keys1 != null && !keys1.isEmpty()) { relData.setIdentifiers(keys1); sources.add(relData); } - log.info("###################### end Import table ["+XMLUtil.getSourceType(relates[i])+"] ####################"); + log.info("###################### end Import table [" + XMLUtil.getSourceType(relates[i]) + + "] ####################"); recur(relates[i], sources); } } - private void setData(Data data,Element element) { + private void relDatatype(Element element, List<Data> sources) throws Exception { + Element[] relates = XMLUtil.getRelateChildNodes(element); + for (int i = 0; i < relates.length; i++) { + log.info("###################### begin import relateddatatype Type [" + + XMLUtil.getSourceType(relates[i]) + "] ####################"); + Data relData = getDataOfRelationType(relates[i]); + sources.add(relData); + log.info("###################### end import relateddatatype Type [" + + XMLUtil.getSourceType(relates[i]) + "] ####################"); + } + } + + // add getDataOfRelationType method + private Data getDataOfRelationType(Element element) { + Data relData = new Data(Constants.RELATION_DATA_TYPE); + setData(relData, element); + String destion = XMLUtil.getDestinationRelationType(element); + relData.setDestinationRelationType(destion); + String sourceRelationType = XMLUtil.getSourceRelationType(element); + relData.setSourceRelationType(sourceRelationType); + String reverse = XMLUtil.getReverseRelationType(element); + relData.setReverse(reverse); + return relData; + } + + private void setData(Data data, Element element) { data.setTableName(XMLUtil.getSourceType(element)); data.setDestinationType(XMLUtil.getDestinationType(element)); data.setRelationType(XMLUtil.getRelateType(element)); data.setRelateTable(XMLUtil.getRelateSourceType(element)); } + private Data getDataOfDataType(Element element) { Data sorData = new Data(Constants.ENTITY_TYPE); - setData(sorData,element); + setData(sorData, element); sorData.setDestinationRelationType(XMLUtil.getDestinationRelationType(element)); return sorData; } @@ -195,8 +271,12 @@ Data relData = new Data(Constants.RELATION_TYPE); setData(relData,element); relData.setDestinationRelationType(XMLUtil.getDestinationRelationType(element)); + //add ############# + String reverse=XMLUtil.getReverseRelationType(element); + relData.setReverse(reverse); return relData; } + private Data getRootData(Element element) { Data rootData = new Data(Constants.ROOT_CATEGORY_TYPE); setData(rootData,element); Index: Data.java =================================================================== RCS file: /var/cvs/contributions/CMSContainer_Modules/dataconversion/src/java/com/finalist/cmsc/dataconversion/service/Data.java,v retrieving revision 1.2 retrieving revision 1.3 diff -u -b -r1.2 -r1.3 --- Data.java 7 Oct 2008 13:05:59 -0000 1.2 +++ Data.java 28 Nov 2008 07:30:44 -0000 1.3 @@ -24,6 +24,8 @@ private String relationType; + private String reverse ; + public Data() { } @@ -110,4 +112,12 @@ public void setRelationType(String relationType) { this.relationType = relationType; } + + public String getReverse() { + return reverse; + } + + public void setReverse(String reverse) { + this.reverse = reverse; + } } Index: NodeService.java =================================================================== RCS file: /var/cvs/contributions/CMSContainer_Modules/dataconversion/src/java/com/finalist/cmsc/dataconversion/service/NodeService.java,v retrieving revision 1.4 retrieving revision 1.5 diff -u -b -r1.4 -r1.5 --- NodeService.java 16 May 2008 11:45:45 -0000 1.4 +++ NodeService.java 28 Nov 2008 07:30:44 -0000 1.5 @@ -1,17 +1,26 @@ package com.finalist.cmsc.dataconversion.service; +import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Properties; + import net.sf.mmapps.commons.bridge.RelationUtil; + import org.mmbase.bridge.Cloud; import org.mmbase.bridge.CloudContext; import org.mmbase.bridge.ContextProvider; +import org.mmbase.bridge.Field; import org.mmbase.bridge.Node; +import org.mmbase.bridge.NodeList; import org.mmbase.bridge.NodeManager; +import org.mmbase.bridge.NodeQuery; import org.mmbase.bridge.Relation; +import org.mmbase.bridge.util.SearchUtil; +import org.mmbase.datatypes.DataType; import org.mmbase.module.core.MMBase; import org.mmbase.module.core.MMObjectNode; import org.mmbase.util.logging.Logger; @@ -87,7 +96,12 @@ Node sourceNode = cloud.getNode(snumber.intValue()); Node desNode = cloud.getNode(dnumber.intValue()); type = data.getDestinationRelationType(); - Relation relate = RelationUtil.createRelation(sourceNode, desNode, data.getDestinationRelationType()); + Relation relate; + if ("true".equals(data.getReverse())) { + relate = RelationUtil.createRelation(desNode, sourceNode, data.getDestinationRelationType()); + }else{ + relate = RelationUtil.createRelation(sourceNode, desNode, data.getDestinationRelationType()); + } if(sourceNode.getNodeManager().getName().equals("contentchannel") && desNode.getNodeManager().getName().equals("article")) { Relation creationRelation = RelationUtil.createRelation(desNode, sourceNode, "creationrel"); creationRelation.commit(); @@ -131,16 +145,57 @@ Integer id = iterator.next(); Integer dId = getDnumber( data, id,sources); Node desNode = cloud.getNode(dId.intValue()); - Relation relate = RelationUtil.createRelation(sourceNode, desNode, data.getDestinationRelationType()); + Relation relate = RelationUtil.createRelation(sourceNode, desNode, type); relate.commit(); } } } catch (Exception e) { - // TODO Auto-generated catch block log.info(String.format("[type %s] [old Node %s] [Node %s ] +"+e.getMessage(),type,key,number)); } return number; } + //createRelationData for the element relateddatatype + public static void createRelationData(List<String> reskeys, Data reldata) { + Cloud cloud = initCloud(); + String type = ""; + try { + if (reldata.getType() == Constants.RELATION_DATA_TYPE) { + for (String sd : reskeys) { + String[] spd = sd.split(","); + Integer snumber = getNewkey(cloud, Integer.parseInt(spd[0])); + Integer dnumber = getNewkey(cloud, Integer.parseInt(spd[1])); + + if (snumber > 0 && dnumber > 0) { + Node sourceNode = cloud.getNode(snumber.intValue()); + Node desNode = cloud.getNode(dnumber.intValue()); + type = reldata.getDestinationRelationType(); + Relation relate; + if ("true".equals(reldata.getReverse())) { + relate = RelationUtil.createRelation(desNode, sourceNode, type); + } else { + relate = RelationUtil.createRelation(sourceNode, desNode, type); + } + relate.commit(); + } + } + } + } catch (Exception e) { + log.info(String.format("[type %s] [old Node %s] [Node %s ] +" + e.getMessage(), type)); + } + } + // search newNumber from mapping + private static Integer getNewkey(Cloud cloud,int parseInt) { + NodeManager manager = cloud.getNodeManager("migration_mappings"); + NodeQuery query = manager.createQuery(); + SearchUtil.addEqualConstraint(query, manager.getField("old_number"), parseInt); + NodeList list = query.getList(); + if(list != null && list.size() >0) { + Node node = (Node)list.get(0); + return node.getIntValue("new_number"); + }else{ + return -1; + } + } /** * get the dnumber from data collection * @param data @@ -210,9 +265,50 @@ node.commit(); } + // insertMigrationMappings to produce a mapping between old and new node numbers. + public static void insertMigrationMappings(List<Data> clonedSources) throws Exception { + Cloud cloud = initCloud(); + NodeManager manager = cloud.getNodeManager("migration_mappings"); + for (Data data : clonedSources) { + if (data.getType() == Constants.ENTITY_TYPE || data.getType() == Constants.ROOT_CATEGORY_TYPE) { + HashMap<Integer, Integer> mappingNumber = data.getIdentifiers(); + for (Iterator iter = mappingNumber.entrySet().iterator(); iter.hasNext();) { + Map.Entry entry = (Map.Entry) iter.next(); + Node node = manager.createNode(); + Object key = entry.getKey(); + Object val = entry.getValue(); + node.setObjectValue("old_number", key); + node.setObjectValue("new_number", val); + node.commit(); + } + } + } + log.info("----> end insertMigrationMappings "); + } + private static Cloud initCloud() { CloudContext context = ContextProvider.getDefaultCloudContext(); return context.getCloud("mmbase"); } + + public static void linkRootDatas(Map<String, ArrayList<Integer>> rootDatas, String node) { + Cloud cloud = initCloud(); + Node desNode = cloud.getNode(Integer.parseInt(node)); + for (Iterator iter = rootDatas.entrySet().iterator(); iter.hasNext();) { + Map.Entry entry = (Map.Entry) iter.next(); + String key = (String) entry.getKey(); + ArrayList<Integer> val = (ArrayList<Integer>) entry.getValue(); + NodeManager manager = cloud.getNodeManager(key); + for (Integer i : val) { + int sn=getNewkey(cloud,i); + if (sn>0) { + Node sourceNode = cloud.getNode(sn); + Relation relate = RelationUtil.createRelation(sourceNode, desNode, "creationrel"); + Relation rel = RelationUtil.createRelation(desNode, sourceNode, "contentrel"); + } + } + } + } + } Index: XMLUtil.java =================================================================== RCS file: /var/cvs/contributions/CMSContainer_Modules/dataconversion/src/java/com/finalist/cmsc/dataconversion/service/XMLUtil.java,v retrieving revision 1.3 retrieving revision 1.4 diff -u -b -r1.3 -r1.4 --- XMLUtil.java 7 Oct 2008 13:05:59 -0000 1.3 +++ XMLUtil.java 28 Nov 2008 07:30:44 -0000 1.4 @@ -145,5 +145,21 @@ return ele; } + public static String getReverseRelationType(Element element) { + return element.getAttribute("reverse"); + } + + public static Element[] getRelateChildNodes(Element element) { + Element[] elements = new Element[0]; + List<Element> list = new ArrayList<Element>(); + NodeList relationList = element.getElementsByTagName("relateddatatype"); + for(int i = 0 ; i < relationList.getLength() ; i++) { + if(((Element)relationList.item(i)).getParentNode() == element) { + list.add((Element)relationList.item(i)); + } + } + return list.toArray(elements); + } + } _______________________________________________ Cvs mailing list Cvs@lists.mmbase.org http://lists.mmbase.org/mailman/listinfo/cvs