BUG Fix concatenation of multiple FastQ files

Adds a test too.

Author: luispedro

ChangeLog

@@ -1,6 +1,7 @@
 Version 1.0.0+
 	* Reintroduce zstd compression (after fixes upstream)
 	* Fix CIGAR interpretation (#109) occurring when I is present
+	* Fix bug with external modules and multiple fastQ inputs
 
 Version 1.0.0 2019-04-24 by luispedro
 	* Fix multiple features usage (#63)

NGLess/Data/FastQ/Utils.hs

@@ -7,37 +7,34 @@ module Data.FastQ.Utils
 import qualified Data.Conduit as C
 import qualified Data.Conduit.Binary as CB
 import           Data.Conduit ((.|))
-import System.IO (hClose)
-import Control.Monad
-import Control.Monad.Except
+import           Control.Monad (forM_)
 
 import Data.List (isSuffixOf)
 
 import NGLess.NGError
 
-import FileManagement
+import FileManagement (makeNGLTempFile)
 import Utils.Conduit (linesC)
-import Data.FastQ
+import Data.FastQ (FastQFilePath(..), fqDecodeC, fqEncodeC)
 import Data.Conduit.Algorithms.Async (asyncGzipTo, conduitPossiblyCompressedFile)
 
 concatenateFQs :: [FastQFilePath] -> NGLessIO FastQFilePath
 concatenateFQs [] = throwShouldNotOccur "Empty argument to concatenateFQs"
 concatenateFQs [f] = return f
 concatenateFQs (FastQFilePath enc fp:rest) = do
-    (fres, h) <- openNGLTempFile "concatenate" fp "fq.gz"
-    let catTo f enc'
-            | enc /= enc' =
-                            conduitPossiblyCompressedFile f
-                                .| linesC
-                                .| fqDecodeC enc'
-                                .| fqEncodeC enc
-                                .| asyncGzipTo h
-            | ".gz" `isSuffixOf` f = CB.sourceFile f .| CB.sinkHandle h
-            | otherwise = conduitPossiblyCompressedFile f .| asyncGzipTo h
-    C.runConduitRes $ catTo fp enc
-    forM_ rest $ \(FastQFilePath enc' f') ->
-        C.runConduitRes (catTo f' enc')
-    liftIO $ hClose h
+    fres <- makeNGLTempFile fp "concatenate" "fq.gz" $ \hout -> do
+        let catTo f enc'
+                | enc /= enc' =
+                                conduitPossiblyCompressedFile f
+                                    .| linesC
+                                    .| fqDecodeC enc'
+                                    .| fqEncodeC enc
+                                    .| asyncGzipTo hout
+                | ".gz" `isSuffixOf` f = CB.sourceFile f .| CB.sinkHandle hout
+                | otherwise = conduitPossiblyCompressedFile f .| asyncGzipTo hout
+        C.runConduitRes $ catTo fp enc
+        forM_ rest $ \(FastQFilePath enc' f') ->
+            C.runConduitRes (catTo f' enc')
     return $ FastQFilePath enc fres
 
 

NGLess/ExternalModules.hs

@@ -27,7 +27,7 @@ import Control.Monad
 import System.Process
 import System.Environment (getEnvironment, getExecutablePath)
 import System.Directory (getDirectoryContents, doesFileExist, doesDirectoryExist)
-import System.Exit
+import System.Exit (ExitCode(..))
 import System.IO
 import System.FilePath
 import Data.Maybe

NGLess/FileManagement.hs

@@ -93,7 +93,10 @@ checkFilenameLength base ext = if len > 240
 -- directory and deleting the file when necessary)
 --
 -- These files will be auto-removed when ngless exits
-openNGLTempFile' :: FilePath -> String -> String -> NGLessIO (ReleaseKey, (FilePath, Handle))
+openNGLTempFile' :: FilePath -- ^ basename
+                        -> String -- ^ prefix
+                        -> String -- ^ extension
+                        -> NGLessIO (ReleaseKey, (FilePath, Handle))
 openNGLTempFile' base prefix ext = do
     tdir <- nConfTemporaryDirectory <$> nglConfiguration
     liftIO $ createDirectoryIfMissing True tdir

docs/sources/whatsnew.rst

@@ -2,6 +2,19 @@
 What's New (History)
 ====================
 
+Post Version 1.0
+----------------
+
+User-visible improvements
+~~~~~~~~~~~~~~~~~~~~~~~~~
+- ZSTD compression is available for output and intermediate files use it for
+  reduced temporary space usage (and possibly faster processing).
+
+Bugfixes
+~~~~~~~~
+
+- Fix bug with external modules and multiple fastQ inputs.
+
 Version 1.0
 -----------
 

tests/exampleExternalModule/example-cmd.ngl

@@ -1,7 +1,8 @@
-ngless '0.0'
+ngless '1.0'
 import "example-cmd" version "0.0"
+import "mocat" version "0.6"
 
-input = fastq('sample.fq')
+input = load_mocat_sample('sample')
 
 testing(input)
 

tests/exampleExternalModule/sample.fq

@@ -0,0 +1 @@
+../../select/sample.fq

tests/exampleExternalModule/sample/sample1.fq

@@ -0,0 +1 @@
+../../select/sample.fq

tests/exampleExternalModule/sample/sample2.fq

@@ -0,0 +1 @@
+../../select/sample.fq