Merge pull request #618 from hartwigmedical/DEV-4061

DEV-4061: Add all OncoAct fields to Compar

compar/src/main/java/com/hartwig/hmftools/compar/Compar.java

@@ -1,14 +1,13 @@
 package com.hartwig.hmftools.compar;
 
 import static java.lang.Math.min;
-import static java.lang.String.format;
 
 import static com.hartwig.hmftools.common.utils.PerformanceCounter.runTimeMinsStr;
 import static com.hartwig.hmftools.compar.ComparConfig.CMP_LOGGER;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.Callable;
-import java.util.stream.Collectors;
 
 import com.google.common.collect.Lists;
 import com.hartwig.hmftools.common.utils.TaskExecutor;
@@ -73,7 +72,7 @@ public void run()
                 ++taskIndex;
             }
 
-            final List<Callable> callableList = sampleTasks.stream().collect(Collectors.toList());
+            final List<Callable> callableList = new ArrayList<>(sampleTasks);
             TaskExecutor.executeTasks(callableList, mConfig.Threads);
         }
         else

compar/src/main/java/com/hartwig/hmftools/compar/ComparConfig.java

@@ -87,6 +87,7 @@ public class ComparConfig
     private boolean mIsValid;
 
     // config strings
+    public static final String GERMLINE_SAMPLE = "germline_sample";
     public static final String CATEGORIES = "categories";
     public static final String MATCH_LEVEL = "match_level";
 
@@ -215,38 +216,70 @@ public String sourceSampleId(final String source, final String sampleId)
 
         if(mapping == null || !mapping.SourceMapping.containsKey(source))
         {
-            CMP_LOGGER.warn("sample({}) source({}) missed mapping", sampleId, source);
             return sampleId;
         }
 
         return mapping.SourceMapping.get(source);
     }
 
+    public String sourceGermlineSampleId(final String source, final String sampleId)
+    {
+        if(!mSampleIdMappings.isEmpty())
+        {
+            SampleIdMapping mapping = mSampleIdMappings.get(sampleId);
+
+            if(mapping != null && mapping.GermlineSourceMapping.containsKey(source))
+            {
+                return mapping.GermlineSourceMapping.get(source);
+            }
+            else if(mapping != null && mapping.GermlineSampleId != null)
+            {
+                return mapping.GermlineSampleId;
+            }
+        }
+        return sourceSampleId(source, sampleId) + "-ref";
+    }
+
     public boolean isValid() { return mIsValid; }
     public boolean singleSample() { return SampleIds.size() == 1; }
     public boolean multiSample() { return SampleIds.size() > 1; }
 
-    private class SampleIdMapping
+    private static class SampleIdMapping
     {
         public final String SampleId;
+        public final String GermlineSampleId;
         public Map<String,String> SourceMapping;
+        public Map<String,String> GermlineSourceMapping;
 
-        public SampleIdMapping(final String sampleId)
+        public SampleIdMapping(final String sampleId, final String germlineSampleId)
         {
             SampleId = sampleId;
+            GermlineSampleId = germlineSampleId;
             SourceMapping = Maps.newHashMap();
+            GermlineSourceMapping = Maps.newHashMap();
         }
     }
 
     private static final String COL_SAMPLE_ID = "SampleId";
+    private static final String COL_GERMLINE_SAMPLE_ID = "GermlineSampleId";
     private static final String COL_REF_SAMPLE_ID = "RefSampleId";
+    private static final String COL_REF_GERMLINE_SAMPLE_ID = "RefGermlineSampleId";
     private static final String COL_NEW_SAMPLE_ID = "NewSampleId";
+    private static final String COL_NEW_GERMLINE_SAMPLE_ID = "NewGermlineSampleId";
 
     private void loadSampleIds(final ConfigBuilder configBuilder)
     {
+        if(configBuilder.hasValue(SAMPLE_ID_FILE) && (configBuilder.hasFlag(SAMPLE) || configBuilder.hasFlag(GERMLINE_SAMPLE)))
+        {
+            CMP_LOGGER.error("when the argument '{}' is set, the arguments '{}' and '{}' should not be set",
+                    SAMPLE_ID_FILE, SAMPLE, GERMLINE_SAMPLE);
+            mIsValid = false;
+            return;
+        }
+
         if(configBuilder.hasValue(SAMPLE))
         {
-            SampleIds.add(configBuilder.getValue(SAMPLE));
+            registerSampleIds(configBuilder.getValue(SAMPLE), configBuilder.getValue(GERMLINE_SAMPLE, null));
             return;
         }
 
@@ -266,8 +299,11 @@ private void loadSampleIds(final ConfigBuilder configBuilder)
             Map<String,Integer> fieldsIndexMap = createFieldsIndexMap(header, CSV_DELIM);
 
             int sampleIndex = fieldsIndexMap.get(COL_SAMPLE_ID);
+            Integer germlineSampleIndex = fieldsIndexMap.get(COL_GERMLINE_SAMPLE_ID);
             Integer refSampleIndex = fieldsIndexMap.get(COL_REF_SAMPLE_ID);
+            Integer refGermlineSampleIndex = fieldsIndexMap.get(COL_REF_GERMLINE_SAMPLE_ID);
             Integer newSampleIndex = fieldsIndexMap.get(COL_NEW_SAMPLE_ID);
+            Integer newGermlineSampleIndex = fieldsIndexMap.get(COL_NEW_GERMLINE_SAMPLE_ID);
 
             for(String line : lines)
             {
@@ -277,23 +313,13 @@ private void loadSampleIds(final ConfigBuilder configBuilder)
                 String[] values = line.split(CSV_DELIM, -1);
 
                 String sampleId = values[sampleIndex];
-
-                SampleIds.add(sampleId);
-
+                String germlineSampleId = germlineSampleIndex != null ? values[germlineSampleIndex] : null;
                 String refSampleId = refSampleIndex != null ? values[refSampleIndex] : null;
+                String refGermlineSampleId = refGermlineSampleIndex != null ? values[refGermlineSampleIndex] : null;
                 String newSampleId = newSampleIndex != null ? values[newSampleIndex] : null;
+                String newGermlineSampleId = newGermlineSampleIndex != null ? values[newGermlineSampleIndex] : null;
 
-                if(refSampleId != null || newSampleId != null)
-                {
-                    SampleIdMapping mapping = new SampleIdMapping(sampleId);
-                    mSampleIdMappings.put(sampleId, mapping);
-
-                    if(refSampleId != null && SourceNames.size() >= 1);
-                        mapping.SourceMapping.put(SourceNames.get(0), refSampleId);
-
-                    if(newSampleId != null && SourceNames.size() >= 2)
-                        mapping.SourceMapping.put(SourceNames.get(1), newSampleId);
-                }
+                registerSampleIds(sampleId, germlineSampleId, refSampleId, refGermlineSampleId, newSampleId, newGermlineSampleId);
             }
 
             CMP_LOGGER.info("loaded {} samples from file", SampleIds.size());
@@ -304,6 +330,30 @@ private void loadSampleIds(final ConfigBuilder configBuilder)
         }
     }
 
+    private void registerSampleIds(final String sampleId, final String germlineSampleId)
+    {
+        registerSampleIds(sampleId, germlineSampleId, null, null, null, null);
+    }
+
+    private void registerSampleIds(final String sampleId, final String germlineSampleId, final String refSampleId,
+            final String refGermlineSampleId, final String newSampleId, final String newGermlineSampleId)
+    {
+        SampleIds.add(sampleId);
+
+        SampleIdMapping mapping = new SampleIdMapping(sampleId, germlineSampleId);
+        mSampleIdMappings.put(sampleId, mapping);
+
+        if(refSampleId != null && SourceNames.size() >= 1)
+            mapping.SourceMapping.put(SourceNames.get(0), refSampleId);
+        if(newSampleId != null && SourceNames.size() >= 2)
+            mapping.SourceMapping.put(SourceNames.get(1), newSampleId);
+
+        if(refGermlineSampleId != null && SourceNames.size() >= 1)
+            mapping.GermlineSourceMapping.put(SourceNames.get(0), refGermlineSampleId);
+        if(newGermlineSampleId != null && SourceNames.size() >= 2)
+            mapping.GermlineSourceMapping.put(SourceNames.get(1), newGermlineSampleId);
+    }
+
     private static String formConfigSourceStr(final String sourceType, final String sourceName)
     {
         return format("%s_%s", sourceType, sourceName);
@@ -373,6 +423,7 @@ public static void addConfig(final ConfigBuilder configBuilder)
                 MATCH_LEVEL, false, "Match level from REPORTABLE (default) or DETAILED", REPORTABLE.toString());
 
         configBuilder.addConfigItem(SAMPLE, SAMPLE_DESC);
+        configBuilder.addConfigItem(GERMLINE_SAMPLE, false, "Sample ID of germline sample if tumor-normal run");
         addSampleIdFile(configBuilder, false);
         configBuilder.addConfigItem(DRIVER_GENE_PANEL_OPTION, DRIVER_GENE_PANEL_OPTION_DESC);
         configBuilder.addConfigItem(THRESHOLDS, "In form: Field,AbsoluteDiff,PercentDiff, separated by ';'");

compar/src/main/java/com/hartwig/hmftools/compar/ComparTask.java

@@ -117,8 +117,9 @@ private Set<String> loadCombinedCopyNumberDriverGenes(final String sampleId)
         for(String sourceName : mConfig.SourceNames)
         {
             String sourceSampleId = mConfig.sourceSampleId(sourceName, sampleId);
+            String sourceGermlineSampleId = mConfig.sourceGermlineSampleId(sourceName, sampleId);
 
-            FileSources fileSources = FileSources.sampleInstance(mConfig.FileSources.get(sourceName), sourceSampleId);
+            FileSources fileSources = FileSources.sampleInstance(mConfig.FileSources.get(sourceName), sourceSampleId, sourceGermlineSampleId);
 
             try
             {

compar/src/main/java/com/hartwig/hmftools/compar/ItemComparer.java

@@ -16,7 +16,7 @@ public interface ItemComparer
 
     List<ComparableItem> loadFromDb(final String sampleId, final DatabaseAccess dbAccess, final String sourceName);
 
-    List<ComparableItem> loadFromFile(final String sampleId, final FileSources fileSources);
+    List<ComparableItem> loadFromFile(final String sampleId, final String germlineSampleId, final FileSources fileSources);
 
     List<String> comparedFieldNames();
 

compar/src/main/java/com/hartwig/hmftools/compar/chord/ChordComparer.java

@@ -62,11 +62,11 @@ public List<ComparableItem> loadFromDb(final String sampleId, final DatabaseAcce
         ChordData chordData = dbAccess.readChord(sampleId);
         final List<ComparableItem> comparableItems = Lists.newArrayList();
         comparableItems.add(new ChordComparData(chordData));
-        return Lists.newArrayList();
+        return comparableItems;
     }
 
     @Override
-    public List<ComparableItem> loadFromFile(final String sampleId, final FileSources fileSources)
+    public List<ComparableItem> loadFromFile(final String sampleId, final String germlineSampleId, final FileSources fileSources)
     {
         final List<ComparableItem> comparableItems = Lists.newArrayList();
 

compar/src/main/java/com/hartwig/hmftools/compar/cider/Cdr3LocusSummaryComparer.java

@@ -60,7 +60,7 @@ public List<ComparableItem> loadFromDb(final String sampleId, final DatabaseAcce
     }
 
     @Override
-    public List<ComparableItem> loadFromFile(final String sampleId, final FileSources fileSources)
+    public List<ComparableItem> loadFromFile(final String sampleId, final String germlineSampleId, final FileSources fileSources)
     {
         try
         {

compar/src/main/java/com/hartwig/hmftools/compar/cider/CiderVdjComparer.java

@@ -50,7 +50,7 @@ public List<ComparableItem> loadFromDb(final String sampleId, final DatabaseAcce
     }
 
     @Override
-    public List<ComparableItem> loadFromFile(final String sampleId, final FileSources fileSources)
+    public List<ComparableItem> loadFromFile(final String sampleId, final String germlineSampleId, final FileSources fileSources)
     {
         try
         {

compar/src/main/java/com/hartwig/hmftools/compar/common/Category.java

@@ -19,6 +19,13 @@ public enum Category
     CUPPA,
     LILAC,
     CHORD,
+    PEACH,
+    VIRUS,
+    TUMOR_FLAGSTAT,
+    GERMLINE_FLAGSTAT,
+    TUMOR_BAM_METRICS,
+    GERMLINE_BAM_METRICS,
+    SNP_GENOTYPE,
     CDR3_SEQUENCE,
     CDR3_LOCUS_SUMMARY,
     TELOMERE_LENGTH;
@@ -35,5 +42,8 @@ public static List<Category> purpleCategories()
 
     public static List<Category> linxCategories() { return Lists.newArrayList(DRIVER, FUSION, DISRUPTION, GERMLINE_SV); }
 
-    public static List<Category> panelCategories() { return Lists.newArrayList(PURITY, DRIVER, SOMATIC_VARIANT, FUSION, DISRUPTION); }
+    public static List<Category> panelCategories()
+    {
+        return Lists.newArrayList(PURITY, DRIVER, SOMATIC_VARIANT, FUSION, DISRUPTION, TUMOR_FLAGSTAT, TUMOR_BAM_METRICS, SNP_GENOTYPE);
+    }
 }

compar/src/main/java/com/hartwig/hmftools/compar/common/CommonUtils.java

@@ -13,12 +13,15 @@
 import static com.hartwig.hmftools.compar.common.MismatchType.NEW_ONLY;
 import static com.hartwig.hmftools.compar.common.MismatchType.REF_ONLY;
 
+import java.io.File;
+import java.nio.file.Files;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import com.hartwig.hmftools.common.genome.chromosome.HumanChromosome;
 import com.hartwig.hmftools.common.genome.refgenome.GenomeLiftoverCache;
 import com.hartwig.hmftools.common.genome.refgenome.RefGenomeVersion;
 import com.hartwig.hmftools.common.region.BasePosition;
@@ -34,20 +37,26 @@
 import com.hartwig.hmftools.compar.linx.DisruptionComparer;
 import com.hartwig.hmftools.compar.linx.FusionComparer;
 import com.hartwig.hmftools.compar.linx.GermlineSvComparer;
+import com.hartwig.hmftools.compar.metrics.GermlineBamMetricsComparer;
+import com.hartwig.hmftools.compar.metrics.GermlineFlagstatComparer;
+import com.hartwig.hmftools.compar.metrics.TumorBamMetricsComparer;
+import com.hartwig.hmftools.compar.metrics.TumorFlagstatComparer;
+import com.hartwig.hmftools.compar.peach.PeachComparer;
 import com.hartwig.hmftools.compar.purple.CopyNumberComparer;
 import com.hartwig.hmftools.compar.purple.GeneCopyNumberComparer;
 import com.hartwig.hmftools.compar.purple.GermlineDeletionComparer;
 import com.hartwig.hmftools.compar.purple.PurityComparer;
 import com.hartwig.hmftools.compar.mutation.GermlineVariantComparer;
 import com.hartwig.hmftools.compar.mutation.SomaticVariantComparer;
+import com.hartwig.hmftools.compar.snpgenotype.SnpGenotypeComparer;
+import com.hartwig.hmftools.compar.virus.VirusComparer;
 import com.hartwig.hmftools.compar.teal.TealComparer;
 
 public class CommonUtils
 {
-    public static final String SUB_ITEM_DELIM = "=";
-
     public static final String FLD_REPORTED = "Reported";
     public static final String FLD_QUAL = "Qual";
+    public static final String FLD_CHROMOSOME_BAND = "ChromosomeBand";
 
     public static List<ItemComparer> buildComparers(final ComparConfig config)
     {
@@ -124,6 +133,27 @@ private static ItemComparer createComparer(final Category category, final Compar
             case GERMLINE_SV:
                 return new GermlineSvComparer(config);
 
+            case PEACH:
+                return new PeachComparer(config);
+
+            case VIRUS:
+                return new VirusComparer(config);
+
+            case TUMOR_FLAGSTAT:
+                return new TumorFlagstatComparer(config);
+
+            case GERMLINE_FLAGSTAT:
+                return new GermlineFlagstatComparer(config);
+
+            case TUMOR_BAM_METRICS:
+                return new TumorBamMetricsComparer(config);
+
+            case GERMLINE_BAM_METRICS:
+                return new GermlineBamMetricsComparer(config);
+
+            case SNP_GENOTYPE:
+                return new SnpGenotypeComparer(config);
+
             case CDR3_SEQUENCE:
                 return new CiderVdjComparer(config);
 
@@ -148,6 +178,7 @@ public static boolean processSample(
         for(String sourceName : config.SourceNames)
         {
             String sourceSampleId = config.sourceSampleId(sourceName, sampleId);
+            String sourceGermlineSampleId = config.sourceGermlineSampleId(sourceName, sampleId);
             List<ComparableItem> items = null;
 
             if(!config.DbConnections.isEmpty())
@@ -156,8 +187,9 @@ public static boolean processSample(
             }
             else
             {
-                FileSources fileSources = FileSources.sampleInstance(config.FileSources.get(sourceName), sourceSampleId);
-                items = comparer.loadFromFile(sourceSampleId, fileSources);
+                FileSources fileSources =
+                        FileSources.sampleInstance(config.FileSources.get(sourceName), sourceSampleId, sourceGermlineSampleId);
+                items = comparer.loadFromFile(sourceSampleId, sourceGermlineSampleId, fileSources);
             }
 
             if(items != null)
@@ -259,4 +291,21 @@ public static BasePosition determineComparisonGenomePosition(
 
         return new BasePosition(chromosome, position);
     }
+
+    public static String determineComparisonChromosome(final String chromosome, final boolean requiresLiftover)
+    {
+        if(requiresLiftover)
+        {
+            return HumanChromosome.fromString(chromosome).name().substring(1);
+        }
+        else
+        {
+            return chromosome;
+        }
+    }
+
+    public static boolean fileExists(final String filename)
+    {
+        return Files.exists(new File(filename).toPath());
+    }
 }

compar/src/main/java/com/hartwig/hmftools/compar/common/DiffThresholds.java

@@ -6,8 +6,6 @@
 import java.util.Map;
 
 import com.google.common.collect.Maps;
-import com.hartwig.hmftools.compar.common.ThresholdData;
-import com.hartwig.hmftools.compar.common.ThresholdType;
 
 public class DiffThresholds
 {