[moses-smt/mosesdecoder] 65ddbf: Implement batched LM requests.

[GitHub <no...@gi...>]

  Branch: refs/heads/master
  Home:   https://github.com/moses-smt/mosesdecoder
  Commit: 65ddbf30b70ed2c92da152ce7b5c1bc43625da01
      https://github.com/moses-smt/mosesdecoder/commit/65ddbf30b70ed2c92da152ce7b5c1bc43625da01
  Author: Oliver Wilson <oli...@ed...>
  Date:   2012-07-02 (Mon, 02 Jul 2012)

  Changed paths:
    M moses/src/Hypothesis.cpp
    M moses/src/Hypothesis.h
    M moses/src/LM/Base.h
    M moses/src/LM/LDHT.cpp
    M moses/src/LMList.h
    M moses/src/Parameter.cpp
    M moses/src/PhraseDictionaryMemory.cpp
    M moses/src/Search.cpp
    M moses/src/SearchNormal.h
    A moses/src/SearchNormalBatch.cpp
    A moses/src/SearchNormalBatch.h
    M moses/src/TypeDef.h

  Log Message:
  -----------
  Implement batched LM requests.


  Commit: 5e624b522c107ddbae4aec17c44d35d6738f1d85
      https://github.com/moses-smt/mosesdecoder/commit/5e624b522c107ddbae4aec17c44d35d6738f1d85
  Author: Oliver Wilson <oli...@ed...>
  Date:   2012-07-02 (Mon, 02 Jul 2012)

  Changed paths:
    M .gitignore
    M BUILD-INSTRUCTIONS.txt
    M Jamroot
    M OnDiskPt/Jamfile
    M OnDiskPt/OnDiskWrapper.cpp
    M OnDiskPt/OnDiskWrapper.h
    M OnDiskPt/Phrase.h
    M OnDiskPt/PhraseNode.cpp
    M OnDiskPt/PhraseNode.h
    M OnDiskPt/SourcePhrase.h
    M OnDiskPt/TargetPhrase.cpp
    M OnDiskPt/TargetPhrase.h
    M OnDiskPt/TargetPhraseCollection.cpp
    M OnDiskPt/TargetPhraseCollection.h
    M OnDiskPt/Vocab.cpp
    M OnDiskPt/Vocab.h
    M OnDiskPt/Word.cpp
    M OnDiskPt/Word.h
    A OnDiskPt/queryOnDiskPt.cpp
    A biconcor/Alignment.cpp
    A biconcor/Alignment.h
    A biconcor/Jamfile
    A biconcor/Mismatch.cpp
    A biconcor/Mismatch.h
    A biconcor/PhrasePair.cpp
    A biconcor/PhrasePair.h
    A biconcor/PhrasePairCollection.cpp
    A biconcor/PhrasePairCollection.h
    A biconcor/SuffixArray.cpp
    A biconcor/SuffixArray.h
    A biconcor/TargetCorpus.cpp
    A biconcor/TargetCorpus.h
    A biconcor/Vocabulary.cpp
    A biconcor/Vocabulary.h
    A biconcor/base64.cpp
    A biconcor/base64.h
    A biconcor/biconcor.cpp
    M contrib/other-builds/CreateOnDisk.xcodeproj/project.pbxproj
    A contrib/other-builds/OnDiskPt/.cproject
    A contrib/other-builds/OnDiskPt/.project
    A contrib/other-builds/kbmira.xcodeproj/project.pbxproj
    M contrib/other-builds/lm.xcodeproj/project.pbxproj
    A contrib/other-builds/lm/.cproject
    A contrib/other-builds/lm/.project
    A contrib/other-builds/mert.xcodeproj/project.pbxproj
    A contrib/other-builds/mert_lib.xcodeproj/project.pbxproj
    M contrib/other-builds/moses-chart-cmd.xcodeproj/project.pbxproj
    M contrib/other-builds/moses-cmd.xcodeproj/project.pbxproj
    A contrib/other-builds/moses-cmd/.cproject
    A contrib/other-builds/moses-cmd/.project
    M contrib/other-builds/moses.xcodeproj/project.pbxproj
    A contrib/other-builds/moses/.cproject
    A contrib/other-builds/moses/.project
    A contrib/other-builds/util/.cproject
    A contrib/other-builds/util/.project
    R contrib/queryOnDiskPt/Jamfile
    R contrib/queryOnDiskPt/compile.sh
    R contrib/queryOnDiskPt/queryOnDiskPt.cpp
    M contrib/sigtest-filter/filter-pt.cpp
    A contrib/tmcombine/test/model3/model/lex.counts.e2f
    A contrib/tmcombine/test/model3/model/lex.counts.f2e
    A contrib/tmcombine/test/model3/model/lex.e2f
    A contrib/tmcombine/test/model3/model/lex.f2e
    A contrib/tmcombine/test/model3/model/phrase-table
    A contrib/tmcombine/test/model4/model/lex.counts.e2f
    A contrib/tmcombine/test/model4/model/lex.counts.f2e
    A contrib/tmcombine/test/model4/model/lex.e2f
    A contrib/tmcombine/test/model4/model/lex.f2e
    A contrib/tmcombine/test/model4/model/phrase-table
    A contrib/tmcombine/test/phrase-table_test10
    M contrib/tmcombine/test/phrase-table_test8
    A contrib/tmcombine/test/phrase-table_test9
    M contrib/tmcombine/tmcombine.py
    M contrib/web/bin/daemon.pl
    M cruise-control/config.ems
    M cruise-control/test_all_new_commits.sh
    M jam-files/sanity.jam
    R kenlm
    M lm/Jamfile
    M lm/bhiksha.cc
    M lm/binary_format.cc
    M lm/build_binary.cc
    M lm/config.cc
    M lm/config.hh
    M lm/left.hh
    M lm/left_test.cc
    M lm/model.cc
    M lm/model.hh
    M lm/model_test.cc
    M lm/model_type.hh
    M lm/ngram_query.cc
    M lm/ngram_query.hh
    M lm/quantize.cc
    M lm/quantize.hh
    M lm/read_arpa.cc
    M lm/read_arpa.hh
    M lm/return.hh
    M lm/search_hashed.cc
    M lm/search_hashed.hh
    M lm/search_trie.cc
    M lm/search_trie.hh
    A lm/state.hh
    M lm/trie.cc
    M lm/trie.hh
    A lm/value.hh
    A lm/value_build.cc
    A lm/value_build.hh
    M lm/vocab.cc
    M lm/vocab.hh
    M lm/weights.hh
    M mert/BleuScorer.cpp
    M mert/BleuScorer.h
    M mert/BleuScorerTest.cpp
    M mert/CderScorer.cpp
    M mert/CderScorer.h
    M mert/Data.cpp
    M mert/Data.h
    M mert/DataTest.cpp
    M mert/Fdstream.h
    M mert/FeatureArray.cpp
    M mert/FeatureArray.h
    M mert/FeatureData.cpp
    M mert/FeatureData.h
    M mert/FeatureDataIterator.cpp
    M mert/FeatureDataIterator.h
    M mert/FeatureDataTest.cpp
    M mert/FeatureStats.cpp
    M mert/FeatureStats.h
    A mert/HypPackEnumerator.cpp
    A mert/HypPackEnumerator.h
    M mert/InterpolatedScorer.cpp
    M mert/InterpolatedScorer.h
    M mert/Jamfile
    M mert/MergeScorer.cpp
    M mert/MergeScorer.h
    A mert/MiraFeatureVector.cpp
    A mert/MiraFeatureVector.h
    A mert/MiraWeightVector.cpp
    A mert/MiraWeightVector.h
    M mert/Ngram.h
    M mert/NgramTest.cpp
    M mert/Optimizer.cpp
    M mert/Optimizer.h
    M mert/OptimizerFactory.cpp
    M mert/OptimizerFactory.h
    M mert/OptimizerFactoryTest.cpp
    M mert/PerScorer.cpp
    M mert/PerScorer.h
    A mert/Permutation.cpp
    A mert/Permutation.h
    A mert/PermutationScorer.cpp
    A mert/PermutationScorer.h
    M mert/Point.cpp
    M mert/Point.h
    M mert/PointTest.cpp
    M mert/PreProcessFilter.cpp
    M mert/PreProcessFilter.h
    M mert/Reference.h
    M mert/ReferenceTest.cpp
    M mert/ScopedVector.h
    M mert/ScoreArray.cpp
    M mert/ScoreArray.h
    M mert/ScoreData.cpp
    M mert/ScoreData.h
    M mert/ScoreDataIterator.cpp
    M mert/ScoreDataIterator.h
    M mert/ScoreStats.cpp
    M mert/ScoreStats.h
    M mert/Scorer.cpp
    M mert/Scorer.h
    M mert/ScorerFactory.cpp
    M mert/ScorerFactory.h
    M mert/SemposOverlapping.cpp
    M mert/SemposOverlapping.h
    M mert/SemposScorer.cpp
    M mert/SemposScorer.h
    A mert/SentenceLevelScorer.cpp
    A mert/SentenceLevelScorer.h
    M mert/Singleton.h
    M mert/SingletonTest.cpp
    A mert/StatisticsBasedScorer.cpp
    A mert/StatisticsBasedScorer.h
    M mert/TerScorer.cpp
    M mert/TerScorer.h
    M mert/Timer.cpp
    M mert/Timer.h
    M mert/TimerTest.cpp
    M mert/Types.h
    M mert/Util.cpp
    M mert/Util.h
    M mert/UtilTest.cpp
    M mert/Vocabulary.cpp
    M mert/Vocabulary.h
    M mert/VocabularyTest.cpp
    M mert/evaluator.cpp
    M mert/extractor.cpp
    R mert/init.opt
    A mert/kbmira.cpp
    M mert/mert.cpp
    M mert/pro.cpp
    M misc/queryPhraseTable.cpp
    M moses-chart-cmd/src/IOWrapper.h
    M moses-chart-cmd/src/Jamfile
    M moses-chart-cmd/src/Main.cpp
    M moses-chart-cmd/src/Main.h
    M moses-chart-cmd/src/TranslationAnalysis.h
    M moses-cmd/src/IOWrapper.cpp
    M moses-cmd/src/IOWrapper.h
    M moses-cmd/src/Jamfile
    M moses-cmd/src/LatticeMBR.cpp
    M moses-cmd/src/LatticeMBR.h
    M moses-cmd/src/LatticeMBRGrid.cpp
    M moses-cmd/src/Main.cpp
    M moses/src/AlignmentInfo.h
    M moses/src/AlignmentInfoCollection.h
    M moses/src/BilingualDynSuffixArray.h
    M moses/src/BitmapContainer.h
    M moses/src/CYKPlusParser/ChartRuleLookupManagerCYKPlus.h
    M moses/src/CYKPlusParser/ChartRuleLookupManagerMemory.h
    M moses/src/CYKPlusParser/ChartRuleLookupManagerOnDisk.h
    M moses/src/CYKPlusParser/DotChart.h
    M moses/src/CYKPlusParser/DotChartInMemory.h
    M moses/src/CYKPlusParser/DotChartOnDisk.h
    M moses/src/CellCollection.h
    M moses/src/ChartCell.h
    M moses/src/ChartCellCollection.h
    M moses/src/ChartCellLabel.h
    M moses/src/ChartCellLabelSet.h
    M moses/src/ChartHypothesis.h
    M moses/src/ChartHypothesisCollection.h
    M moses/src/ChartManager.h
    M moses/src/ChartRuleLookupManager.h
    M moses/src/ChartTranslationOption.h
    M moses/src/ChartTranslationOptionCollection.h
    M moses/src/ChartTranslationOptionList.h
    M moses/src/ChartTrellisDetour.h
    M moses/src/ChartTrellisDetourQueue.h
    M moses/src/ChartTrellisNode.h
    M moses/src/ChartTrellisPath.h
    M moses/src/ConfusionNet.h
    M moses/src/DecodeFeature.h
    M moses/src/DecodeGraph.h
    M moses/src/DecodeStep.h
    M moses/src/Dictionary.h
    M moses/src/DummyScoreProducers.h
    M moses/src/DynSAInclude/RandLMCache.h
    M moses/src/DynSAInclude/RandLMFilter.h
    M moses/src/DynSAInclude/file.h
    M moses/src/DynSAInclude/hash.h
    M moses/src/DynSAInclude/onlineRLM.h
    M moses/src/DynSAInclude/params.h
    M moses/src/DynSAInclude/perfectHash.h
    M moses/src/DynSAInclude/quantizer.h
    M moses/src/DynSAInclude/utils.h
    M moses/src/DynSAInclude/vocab.h
    M moses/src/DynSuffixArray.h
    M moses/src/FFState.h
    M moses/src/Factor.h
    M moses/src/FactorCollection.h
    M moses/src/FactorTypeSet.h
    M moses/src/FeatureFunction.h
    M moses/src/FilePtr.h
    M moses/src/GlobalLexicalModel.h
    M moses/src/HypoList.h
    M moses/src/HypothesisStack.h
    M moses/src/HypothesisStackCubePruning.h
    M moses/src/HypothesisStackNormal.h
    M moses/src/InputType.h
    M moses/src/LM/IRST.h
    M moses/src/LM/Implementation.cpp
    M moses/src/LM/Implementation.h
    M moses/src/LM/Joint.h
    M moses/src/LM/Ken.cpp
    M moses/src/LM/Ken.h
    M moses/src/LM/MultiFactor.h
    M moses/src/LM/ORLM.h
    M moses/src/LM/ParallelBackoff.h
    M moses/src/LM/Remote.h
    M moses/src/LM/SRI.h
    M moses/src/LMList.h
    M moses/src/LVoc.h
    M moses/src/LexicalReordering.h
    M moses/src/LexicalReorderingState.h
    M moses/src/LexicalReorderingTable.h
    M moses/src/Manager.h
    M moses/src/NonTerminal.h
    M moses/src/OutputCollector.h
    M moses/src/PCNTools.h
    M moses/src/PDTAimp.h
    M moses/src/Parameter.h
    M moses/src/PartialTranslOptColl.h
    M moses/src/Phrase.h
    M moses/src/PhraseDictionaryDynSuffixArray.h
    M moses/src/PhraseDictionaryNode.h
    M moses/src/PhraseDictionaryTree.h
    M moses/src/PhraseDictionaryTreeAdaptor.h
    M moses/src/PrefixTree.h
    M moses/src/PrefixTreeMap.h
    M moses/src/ReorderingConstraint.h
    M moses/src/ReorderingStack.h
    M moses/src/RuleCube.h
    M moses/src/RuleCubeItem.h
    M moses/src/RuleCubeQueue.h
    M moses/src/RuleTable/Loader.h
    M moses/src/RuleTable/LoaderCompact.h
    M moses/src/RuleTable/LoaderFactory.cpp
    M moses/src/RuleTable/LoaderFactory.h
    M moses/src/RuleTable/LoaderHiero.h
    M moses/src/RuleTable/LoaderStandard.h
    M moses/src/RuleTable/PhraseDictionaryALSuffixArray.cpp
    M moses/src/RuleTable/PhraseDictionaryALSuffixArray.h
    M moses/src/RuleTable/PhraseDictionaryNodeSCFG.h
    M moses/src/RuleTable/PhraseDictionaryOnDisk.cpp
    M moses/src/RuleTable/PhraseDictionaryOnDisk.h
    M moses/src/RuleTable/PhraseDictionarySCFG.h
    M moses/src/RuleTable/Trie.h
    M moses/src/RuleTable/UTrie.h
    M moses/src/RuleTable/UTrieNode.h
    M moses/src/Scope3Parser/ApplicableRuleTrie.h
    M moses/src/Scope3Parser/IntermediateVarSpanNode.h
    M moses/src/Scope3Parser/Parser.h
    M moses/src/Scope3Parser/StackLattice.h
    M moses/src/Scope3Parser/StackLatticeBuilder.h
    M moses/src/Scope3Parser/VarSpanNode.h
    M moses/src/Scope3Parser/VarSpanTrieBuilder.h
    M moses/src/ScoreIndexManager.h
    M moses/src/Search.h
    M moses/src/SearchCubePruning.h
    M moses/src/SearchNormal.h
    M moses/src/Sentence.h
    M moses/src/SentenceStats.h
    M moses/src/SquareMatrix.h
    M moses/src/StaticData.h
    M moses/src/ThreadPool.h
    M moses/src/Timer.h
    M moses/src/TranslationOptionCollection.h
    M moses/src/TranslationOptionCollectionConfusionNet.h
    M moses/src/TranslationOptionCollectionText.h
    M moses/src/TranslationOptionList.h
    M moses/src/TreeInput.h
    M moses/src/TrellisPath.h
    M moses/src/TrellisPathCollection.h
    M moses/src/TrellisPathList.h
    M moses/src/TypeDef.h
    M moses/src/UniqueObject.h
    M moses/src/UserMessage.h
    M moses/src/Util.h
    M moses/src/Word.cpp
    M moses/src/Word.h
    M moses/src/WordLattice.h
    M moses/src/gzfilebuf.h
    A phrase-extract/AlignmentPhrase.cpp
    A phrase-extract/AlignmentPhrase.h
    A phrase-extract/ExtractedRule.cpp
    A phrase-extract/ExtractedRule.h
    A phrase-extract/Hole.h
    A phrase-extract/HoleCollection.cpp
    A phrase-extract/HoleCollection.h
    A phrase-extract/InputFileStream.cpp
    A phrase-extract/InputFileStream.h
    A phrase-extract/Jamfile
    A phrase-extract/OutputFileStream.cpp
    A phrase-extract/OutputFileStream.h
    A phrase-extract/PhraseAlignment.cpp
    A phrase-extract/PhraseAlignment.h
    A phrase-extract/RuleExist.h
    A phrase-extract/RuleExtractionOptions.h
    A phrase-extract/SafeGetline.h
    A phrase-extract/SentenceAlignment.cpp
    A phrase-extract/SentenceAlignment.h
    A phrase-extract/SentenceAlignmentWithSyntax.cpp
    A phrase-extract/SentenceAlignmentWithSyntax.h
    A phrase-extract/SyntaxTree.cpp
    A phrase-extract/SyntaxTree.h
    A phrase-extract/XmlException.h
    A phrase-extract/XmlTree.cpp
    A phrase-extract/XmlTree.h
    A phrase-extract/consolidate-direct.cpp
    A phrase-extract/consolidate-direct.vcxproj
    A phrase-extract/consolidate-reverse.cpp
    A phrase-extract/consolidate.cpp
    A phrase-extract/consolidate.vcxproj
    A phrase-extract/extract-ghkm/Alignment.cpp
    A phrase-extract/extract-ghkm/Alignment.h
    A phrase-extract/extract-ghkm/AlignmentGraph.cpp
    A phrase-extract/extract-ghkm/AlignmentGraph.h
    A phrase-extract/extract-ghkm/ComposedRule.cpp
    A phrase-extract/extract-ghkm/ComposedRule.h
    A phrase-extract/extract-ghkm/Exception.h
    A phrase-extract/extract-ghkm/ExtractGHKM.cpp
    A phrase-extract/extract-ghkm/ExtractGHKM.h
    A phrase-extract/extract-ghkm/Jamfile
    A phrase-extract/extract-ghkm/Main.cpp
    A phrase-extract/extract-ghkm/Node.cpp
    A phrase-extract/extract-ghkm/Node.h
    A phrase-extract/extract-ghkm/Options.h
    A phrase-extract/extract-ghkm/ParseTree.cpp
    A phrase-extract/extract-ghkm/ParseTree.h
    A phrase-extract/extract-ghkm/ScfgRule.cpp
    A phrase-extract/extract-ghkm/ScfgRule.h
    A phrase-extract/extract-ghkm/ScfgRuleWriter.cpp
    A phrase-extract/extract-ghkm/ScfgRuleWriter.h
    A phrase-extract/extract-ghkm/Span.cpp
    A phrase-extract/extract-ghkm/Span.h
    A phrase-extract/extract-ghkm/Subgraph.cpp
    A phrase-extract/extract-ghkm/Subgraph.h
    A phrase-extract/extract-ghkm/XmlTreeParser.cpp
    A phrase-extract/extract-ghkm/XmlTreeParser.h
    A phrase-extract/extract-lex.cpp
    A phrase-extract/extract-lex.h
    A phrase-extract/extract-lex.vcxproj
    A phrase-extract/extract-rules.cpp
    A phrase-extract/extract-rules.vcxproj
    A phrase-extract/extract.cpp
    A phrase-extract/extract.vcxproj
    A phrase-extract/extract.xcodeproj/project.pbxproj
    A phrase-extract/gzfilebuf.h
    A phrase-extract/hierarchical.h
    A phrase-extract/lexical-reordering/InputFileStream.cpp
    A phrase-extract/lexical-reordering/InputFileStream.h
    A phrase-extract/lexical-reordering/Jamfile
    A phrase-extract/lexical-reordering/gzfilebuf.h
    A phrase-extract/lexical-reordering/reordering_classes.cpp
    A phrase-extract/lexical-reordering/reordering_classes.h
    A phrase-extract/lexical-reordering/score.cpp
    A phrase-extract/pcfg-common/Jamfile
    A phrase-extract/pcfg-common/exception.h
    A phrase-extract/pcfg-common/numbered_set.h
    A phrase-extract/pcfg-common/pcfg.cc
    A phrase-extract/pcfg-common/pcfg.h
    A phrase-extract/pcfg-common/pcfg_tree.h
    A phrase-extract/pcfg-common/syntax_tree.h
    A phrase-extract/pcfg-common/tool.cc
    A phrase-extract/pcfg-common/tool.h
    A phrase-extract/pcfg-common/typedef.h
    A phrase-extract/pcfg-common/xml_tree_parser.cc
    A phrase-extract/pcfg-common/xml_tree_parser.h
    A phrase-extract/pcfg-common/xml_tree_writer.h
    A phrase-extract/pcfg-extract/Jamfile
    A phrase-extract/pcfg-extract/main.cc
    A phrase-extract/pcfg-extract/options.h
    A phrase-extract/pcfg-extract/pcfg_extract.cc
    A phrase-extract/pcfg-extract/pcfg_extract.h
    A phrase-extract/pcfg-extract/rule_collection.cc
    A phrase-extract/pcfg-extract/rule_collection.h
    A phrase-extract/pcfg-extract/rule_extractor.cc
    A phrase-extract/pcfg-extract/rule_extractor.h
    A phrase-extract/pcfg-score/Jamfile
    A phrase-extract/pcfg-score/main.cc
    A phrase-extract/pcfg-score/options.h
    A phrase-extract/pcfg-score/pcfg_score.cc
    A phrase-extract/pcfg-score/pcfg_score.h
    A phrase-extract/pcfg-score/tree_scorer.cc
    A phrase-extract/pcfg-score/tree_scorer.h
    A phrase-extract/phrase-extract.sln
    A phrase-extract/relax-parse.cpp
    A phrase-extract/relax-parse.h
    A phrase-extract/score.cpp
    A phrase-extract/score.h
    A phrase-extract/score.vcxproj
    A phrase-extract/statistics.cpp
    A phrase-extract/tables-core.cpp
    A phrase-extract/tables-core.h
    M regression-testing/Jamfile
    M regression-testing/MosesRegressionTesting.pm
    M regression-testing/run-test-extract.perl
    M regression-testing/run-test-mert.perl
    R regression-testing/run-test-suite.perl
    M scripts/Jamfile
    M scripts/analysis/weight-scan.pl
    R scripts/ems/biconcor/Alignment.cpp
    R scripts/ems/biconcor/Alignment.h
    R scripts/ems/biconcor/Jamfile
    R scripts/ems/biconcor/Mismatch.cpp
    R scripts/ems/biconcor/Mismatch.h
    R scripts/ems/biconcor/PhrasePair.cpp
    R scripts/ems/biconcor/PhrasePair.h
    R scripts/ems/biconcor/PhrasePairCollection.cpp
    R scripts/ems/biconcor/PhrasePairCollection.h
    R scripts/ems/biconcor/SuffixArray.cpp
    R scripts/ems/biconcor/SuffixArray.h
    R scripts/ems/biconcor/TargetCorpus.cpp
    R scripts/ems/biconcor/TargetCorpus.h
    R scripts/ems/biconcor/Vocabulary.cpp
    R scripts/ems/biconcor/Vocabulary.h
    R scripts/ems/biconcor/base64.cpp
    R scripts/ems/biconcor/base64.h
    R scripts/ems/biconcor/biconcor.cpp
    M scripts/ems/example/config.basic
    M scripts/ems/example/config.factored
    M scripts/ems/example/config.hierarchical
    M scripts/ems/example/config.syntax
    M scripts/ems/example/config.toy
    M scripts/ems/experiment.machines
    M scripts/ems/experiment.meta
    M scripts/ems/experiment.perl
    M scripts/ems/support/generic-multicore-parallelizer.perl
    M scripts/ems/support/report-experiment-scores.perl
    M scripts/ems/support/run-command-on-multiple-refsets.perl
    M scripts/ems/support/split-sentences.perl
    M scripts/generic/compound-splitter.perl
    M scripts/generic/extract-parallel.perl
    A scripts/generic/score-parallel.perl
    M scripts/generic/trainlm-irst.perl
    M scripts/recaser/train-recaser.perl
    A scripts/share/nonbreaking_prefixes/README.txt
    A scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ca
    A scripts/share/nonbreaking_prefixes/nonbreaking_prefix.de
    A scripts/share/nonbreaking_prefixes/nonbreaking_prefix.el
    A scripts/share/nonbreaking_prefixes/nonbreaking_prefix.en
    A scripts/share/nonbreaking_prefixes/nonbreaking_prefix.es
    A scripts/share/nonbreaking_prefixes/nonbreaking_prefix.fr
    A scripts/share/nonbreaking_prefixes/nonbreaking_prefix.is
    A scripts/share/nonbreaking_prefixes/nonbreaking_prefix.it
    A scripts/share/nonbreaking_prefixes/nonbreaking_prefix.nl
    A scripts/share/nonbreaking_prefixes/nonbreaking_prefix.pl
    A scripts/share/nonbreaking_prefixes/nonbreaking_prefix.pt
    A scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ro
    A scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ru
    A scripts/share/nonbreaking_prefixes/nonbreaking_prefix.sk
    A scripts/share/nonbreaking_prefixes/nonbreaking_prefix.sl
    A scripts/share/nonbreaking_prefixes/nonbreaking_prefix.sv
    M scripts/tokenizer/deescape-special-chars.perl
    M scripts/tokenizer/detokenizer.perl
    M scripts/tokenizer/escape-special-chars.perl
    R scripts/tokenizer/nonbreaking_prefixes/README.txt
    R scripts/tokenizer/nonbreaking_prefixes/nonbreaking_prefix.ca
    R scripts/tokenizer/nonbreaking_prefixes/nonbreaking_prefix.de
    R scripts/tokenizer/nonbreaking_prefixes/nonbreaking_prefix.el
    R scripts/tokenizer/nonbreaking_prefixes/nonbreaking_prefix.en
    R scripts/tokenizer/nonbreaking_prefixes/nonbreaking_prefix.es
    R scripts/tokenizer/nonbreaking_prefixes/nonbreaking_prefix.fr
    R scripts/tokenizer/nonbreaking_prefixes/nonbreaking_prefix.is
    R scripts/tokenizer/nonbreaking_prefixes/nonbreaking_prefix.it
    R scripts/tokenizer/nonbreaking_prefixes/nonbreaking_prefix.nl
    R scripts/tokenizer/nonbreaking_prefixes/nonbreaking_prefix.pl
    R scripts/tokenizer/nonbreaking_prefixes/nonbreaking_prefix.pt
    R scripts/tokenizer/nonbreaking_prefixes/nonbreaking_prefix.ro
    R scripts/tokenizer/nonbreaking_prefixes/nonbreaking_prefix.ru
    R scripts/tokenizer/nonbreaking_prefixes/nonbreaking_prefix.sk
    R scripts/tokenizer/nonbreaking_prefixes/nonbreaking_prefix.sl
    R scripts/tokenizer/nonbreaking_prefixes/nonbreaking_prefix.sv
    M scripts/tokenizer/tokenizer.perl
    M scripts/training/Jamfile
    R scripts/training/compact-rule-table/Compactify.cpp
    R scripts/training/compact-rule-table/Compactify.h
    R scripts/training/compact-rule-table/Compactify_Main.cpp
    R scripts/training/compact-rule-table/Exception.h
    R scripts/training/compact-rule-table/Jamfile
    R scripts/training/compact-rule-table/NumberedSet.h
    R scripts/training/compact-rule-table/Options.h
    R scripts/training/compact-rule-table/RuleTableParser.cpp
    R scripts/training/compact-rule-table/RuleTableParser.h
    R scripts/training/compact-rule-table/Tool.h
    M scripts/training/filter-model-given-input.pl
    A scripts/training/giza2bal.pl
    R scripts/training/lexical-reordering/InputFileStream.cpp
    R scripts/training/lexical-reordering/InputFileStream.h
    R scripts/training/lexical-reordering/Jamfile
    R scripts/training/lexical-reordering/gzfilebuf.h
    R scripts/training/lexical-reordering/reordering_classes.cpp
    R scripts/training/lexical-reordering/reordering_classes.h
    R scripts/training/lexical-reordering/score.cpp
    M scripts/training/mert-moses-multi.pl
    M scripts/training/mert-moses.pl
    R scripts/training/phrase-extract/AlignmentPhrase.cpp
    R scripts/training/phrase-extract/AlignmentPhrase.h
    R scripts/training/phrase-extract/ExtractedRule.cpp
    R scripts/training/phrase-extract/ExtractedRule.h
    R scripts/training/phrase-extract/Hole.h
    R scripts/training/phrase-extract/HoleCollection.cpp
    R scripts/training/phrase-extract/HoleCollection.h
    R scripts/training/phrase-extract/InputFileStream.cpp
    R scripts/training/phrase-extract/InputFileStream.h
    R scripts/training/phrase-extract/Jamfile
    R scripts/training/phrase-extract/OutputFileStream.cpp
    R scripts/training/phrase-extract/OutputFileStream.h
    R scripts/training/phrase-extract/PhraseAlignment.cpp
    R scripts/training/phrase-extract/PhraseAlignment.h
    R scripts/training/phrase-extract/RuleExist.h
    R scripts/training/phrase-extract/RuleExtractionOptions.h
    R scripts/training/phrase-extract/SafeGetline.h
    R scripts/training/phrase-extract/SentenceAlignment.cpp
    R scripts/training/phrase-extract/SentenceAlignment.h
    R scripts/training/phrase-extract/SentenceAlignmentWithSyntax.cpp
    R scripts/training/phrase-extract/SentenceAlignmentWithSyntax.h
    R scripts/training/phrase-extract/SyntaxTree.cpp
    R scripts/training/phrase-extract/SyntaxTree.h
    R scripts/training/phrase-extract/XmlException.h
    R scripts/training/phrase-extract/XmlTree.cpp
    R scripts/training/phrase-extract/XmlTree.h
    R scripts/training/phrase-extract/consolidate-direct.cpp
    R scripts/training/phrase-extract/consolidate-direct.vcxproj
    R scripts/training/phrase-extract/consolidate-reverse.cpp
    R scripts/training/phrase-extract/consolidate.cpp
    R scripts/training/phrase-extract/consolidate.vcxproj
    R scripts/training/phrase-extract/extract-ghkm/Alignment.cpp
    R scripts/training/phrase-extract/extract-ghkm/Alignment.h
    R scripts/training/phrase-extract/extract-ghkm/AlignmentGraph.cpp
    R scripts/training/phrase-extract/extract-ghkm/AlignmentGraph.h
    R scripts/training/phrase-extract/extract-ghkm/ComposedRule.cpp
    R scripts/training/phrase-extract/extract-ghkm/ComposedRule.h
    R scripts/training/phrase-extract/extract-ghkm/Exception.h
    R scripts/training/phrase-extract/extract-ghkm/ExtractGHKM.cpp
    R scripts/training/phrase-extract/extract-ghkm/ExtractGHKM.h
    R scripts/training/phrase-extract/extract-ghkm/Jamfile
    R scripts/training/phrase-extract/extract-ghkm/Main.cpp
    R scripts/training/phrase-extract/extract-ghkm/Node.cpp
    R scripts/training/phrase-extract/extract-ghkm/Node.h
    R scripts/training/phrase-extract/extract-ghkm/Options.h
    R scripts/training/phrase-extract/extract-ghkm/ParseTree.cpp
    R scripts/training/phrase-extract/extract-ghkm/ParseTree.h
    R scripts/training/phrase-extract/extract-ghkm/ScfgRule.cpp
    R scripts/training/phrase-extract/extract-ghkm/ScfgRule.h
    R scripts/training/phrase-extract/extract-ghkm/ScfgRuleWriter.cpp
    R scripts/training/phrase-extract/extract-ghkm/ScfgRuleWriter.h
    R scripts/training/phrase-extract/extract-ghkm/Span.cpp
    R scripts/training/phrase-extract/extract-ghkm/Span.h
    R scripts/training/phrase-extract/extract-ghkm/Subgraph.cpp
    R scripts/training/phrase-extract/extract-ghkm/Subgraph.h
    R scripts/training/phrase-extract/extract-ghkm/XmlTreeParser.cpp
    R scripts/training/phrase-extract/extract-ghkm/XmlTreeParser.h
    R scripts/training/phrase-extract/extract-lex.cpp
    R scripts/training/phrase-extract/extract-lex.h
    R scripts/training/phrase-extract/extract-lex.vcxproj
    R scripts/training/phrase-extract/extract-rules.cpp
    R scripts/training/phrase-extract/extract-rules.vcxproj
    R scripts/training/phrase-extract/extract.cpp
    R scripts/training/phrase-extract/extract.vcxproj
    R scripts/training/phrase-extract/extract.xcodeproj/project.pbxproj
    R scripts/training/phrase-extract/gzfilebuf.h
    R scripts/training/phrase-extract/hierarchical.h
    R scripts/training/phrase-extract/phrase-extract.sln
    R scripts/training/phrase-extract/relax-parse.cpp
    R scripts/training/phrase-extract/relax-parse.h
    R scripts/training/phrase-extract/score.cpp
    R scripts/training/phrase-extract/score.h
    R scripts/training/phrase-extract/score.vcxproj
    R scripts/training/phrase-extract/statistics.cpp
    R scripts/training/phrase-extract/tables-core.cpp
    R scripts/training/phrase-extract/tables-core.h
    R scripts/training/symal/Jamfile
    R scripts/training/symal/cmd.c
    R scripts/training/symal/cmd.h
    R scripts/training/symal/giza2bal.pl
    R scripts/training/symal/symal.cpp
    R scripts/training/symal/symal.vcproj
    A scripts/training/train-model.perl
    R scripts/training/train-model.perl.missing_bin_dir
    A scripts/training/wrappers/adam-suffix-array/suffix-array-create.sh
    A scripts/training/wrappers/adam-suffix-array/suffix-array-extract.sh
    A scripts/training/wrappers/berkeleyparsed2mosesxml.perl
    A scripts/training/wrappers/mosesxml2berkeleyparsed.perl
    A scripts/training/wrappers/parse-de-berkeley.perl
    M scripts/training/wrappers/parse-de-bitpar.perl
    R scripts/training/wrappers/suffix-array-create.sh
    R scripts/training/wrappers/suffix-array-extract.sh
    A scripts/training/wrappers/syntax-hyphen-splitting.perl
    M scripts/training/zmert-moses.pl
    A symal/Jamfile
    A symal/cmd.c
    A symal/cmd.h
    A symal/symal.cpp
    A symal/symal.vcproj
    M util/Jamfile
    M util/bit_packing.hh
    M util/ersatz_progress.cc
    M util/ersatz_progress.hh
    M util/file.cc
    M util/file_piece.cc
    M util/have.hh
    M util/mmap.cc
    M util/murmur_hash.cc
    M util/murmur_hash.hh
    M util/probing_hash_table.hh
    A util/usage.cc
    A util/usage.hh

  Log Message:
  -----------
  Merge branch 'master' into asynclm


Compare: https://github.com/moses-smt/mosesdecoder/compare/b5aa04feb78d...5e624b522c10