Merge branch 'release/0.9.0'

Author: tshauck

.yeyo.yaml

@@ -1,4 +1,4 @@
-version: 0.8.0
+version: 0.9.0
 tag_template: v{{ yeyo_version }}
 commit_template: v{{ yeyo_version }}
 files:

CHANGELOG.md

@@ -2,6 +2,23 @@
 
 ## Development
 
+### Added
+
+- Parser now outputs the length of the tensor not including padding. This is
+  useful for packing and length based iteration.
+- Generating masked output from the parse_record method is now available.
+- Alphabet can include an optional mask token.
+
+### Changed
+
+- Can now specify how large of kmer step size to generate when supplying a kmer
+  value.
+- Renames EncodedSeq.integer_encoded to EncodedSeq.get_integer_encoding which
+  takes a kmer_step_size to specify how large of steps to take when encoding.
+- Add parsed_seq_len to the SequenceParser object to control how much padding to
+  apply to the end of the integer encoded sequence. This is useful since a batch
+  of tensors is expected to have the same size.
+
 ## 0.8.0 (2019-07-04)
 
 ### Fixed

Makefile

@@ -67,3 +67,7 @@ test_integration:
 .PHONY: test_unit
 test_unit:
 	pytest --cov-report term-missing --cov=gcgc -m 'not integration'
+
+.PHONY: fmt
+fmt:
+	black .

VERSION

@@ -1 +1 @@
-0.8.0
+0.9.0

docs/index.md

@@ -39,4 +39,4 @@ $ pip install gcgc[torch]
 
 ## Documentation Version
 
-The documentation you're reading was build for version: `0.8.0`.
+The documentation you're reading was build for version: `0.9.0`.

gcgc/__init__.py

@@ -2,4 +2,4 @@
 # All Rights Reserved
 """Top-level GCGC module."""
 
-__version__ = "0.8.0"
+__version__ = "0.9.0"

gcgc/alphabet/base.py

@@ -4,6 +4,7 @@
 
 import itertools as it
 from typing import Iterable
+from typing import Optional
 from typing import Sequence
 
 from gcgc.exceptions import GCGCAlphabetLetterEncodingException
@@ -15,16 +16,24 @@ class EncodingAlphabet:
     PADDING: str = "|"
     START: str = ">"
     END: str = "<"
+    MASK: str = "#"
 
     # Convince linting that EncodingAlphabet will have a letters attribute.
     letters: str
 
-    def __init__(self, kmer_size: int = 1, start_token: bool = True, end_token: bool = True):
+    def __init__(
+        self,
+        kmer_size: int = 1,
+        start_token: bool = True,
+        end_token: bool = True,
+        masked: bool = False,
+    ):
         """Create the EncodingAlphabet object."""
 
         self.start = start_token
         self.end = end_token
         self.kmer_size = kmer_size
+        self.masked = masked
 
         self.encoding_index = {letter: idx for idx, letter in enumerate(self.kmers_and_tokens)}
         self.decoding_index = {idx: letter for letter, idx in self.encoding_index.items()}
@@ -42,6 +51,8 @@ def tokens(self):
             append_string.append(self.START)
         if self.end:
             append_string.append(self.END)
+        if self.masked:
+            append_string.append(self.MASK)
 
         return "".join(append_string)
 
@@ -64,6 +75,11 @@ def encoded_start(self):
         """Get the integer for the start character."""
         return self.encode_token(self.START)
 
+    @property
+    def encoded_mask(self):
+        """Get the integer for the mask character."""
+        return self.encode_token(self.MASK)
+
     @property
     def encoded_end(self):
         """Get the integer for the end character."""
@@ -98,23 +114,31 @@ def _kmer_one(self, seq):
         except KeyError:
             raise GCGCAlphabetLetterEncodingException(f"{kmer} not in {self.encoding_index}")
 
-    def _kmer_n(self, seq: str) -> Sequence[int]:
+    def _kmer_n(self, seq: str, kmer_step_size: int) -> Sequence[int]:
         try:
             encoded = []
 
             seq_len = len(seq)
             iterations = seq_len - self.kmer_size + 1
 
-            for i in range(0, iterations):
+            for i in range(0, iterations, kmer_step_size):
                 kmer = seq[i : i + self.kmer_size]
                 encoded.append(self.encoding_index[kmer])
             return encoded
 
         except KeyError:
             raise GCGCAlphabetLetterEncodingException(f"{kmer} not in {self.encoding_index}")
 
-    def integer_encode(self, seq: str) -> Sequence[int]:
-        """Integer encode the sequence."""
+    def integer_encode(self, seq: str, kmer_step_size: Optional[int] = None) -> Sequence[int]:
+        """Integer encode the sequence.
+
+        Args:
+            seq: The sequence to encode.
+            kmer_step_size: The size of the kmer step, if None uses self.kmer
+
+        Returns:
+            The list of integers that represent the sequence.
+        """
 
         stripped_seq = "".join(s for s in seq if s not in {self.START, self.END, self.PADDING})
         seq_len = len(stripped_seq)
@@ -127,10 +151,11 @@ def integer_encode(self, seq: str) -> Sequence[int]:
         if self.kmer_size == 1:
             encoded_seq = self._kmer_one(stripped_seq)
         else:
-            encoded_seq = self._kmer_n(stripped_seq)
+            passed_kmer_step_size = kmer_step_size if kmer_step_size is not None else self.kmer_size
+            encoded_seq = self._kmer_n(stripped_seq, passed_kmer_step_size)
 
-        if seq[0] == self.START:
-            encoded_seq = [self.encoding_index[self.START]] + encoded_seq
+        if seq.startswith(self.START):
+            encoded_seq = [self.encoded_start] + encoded_seq
 
         non_seq_ending = "".join(s for s in seq if s in {self.END, self.PADDING})
         if non_seq_ending:

gcgc/encoded_seq/encoded_seq.py

@@ -2,10 +2,9 @@
 # All Rights Reserved
 """Contains the EncodedSeq object."""
 
-from typing import Iterable, Sequence, Union
+from typing import Iterable, Union, Optional
 
 from Bio.Seq import Seq
-import numpy as np
 
 from gcgc.alphabet.base import EncodingAlphabet
 from gcgc.alphabet.utils import biopython_alphabet_to_gcgc_alphabet
@@ -117,10 +116,9 @@ def shift(self, offset: int) -> "EncodedSeq":
 
         raise ValueError(f"Unsure how to handle {offset}.")
 
-    @property
-    def integer_encoded(self):
+    def get_integer_encoding(self, kmer_step_size: Optional[int] = None):
         """Return the underlying sequence in its integer representation."""
-        return self.alphabet.integer_encode(self)
+        return self.alphabet.integer_encode(self, kmer_step_size)
 
     @classmethod
     def from_integer_encoded_seq(

gcgc/ml/pytorch_utils/data.py

@@ -3,6 +3,7 @@
 """Objects and methods for dealing with PyTorch data."""
 
 from pathlib import Path
+from typing import Optional
 from typing import Sequence
 
 from Bio import File
@@ -19,11 +20,17 @@
 class GenomicDataset(torch.utils.data.Dataset):
     """GenomicDataset can be used to load sequence information into a format aminable to PyTorch."""
 
-    def __init__(self, file_index: File._SQLiteManySeqFilesDict, parser: TorchSequenceParser):
+    def __init__(
+        self,
+        file_index: File._SQLiteManySeqFilesDict,
+        parser: TorchSequenceParser,
+        parsed_seq_len: Optional[int] = None,
+    ):
         """Initialize the GenomicDataset object."""
 
         self._file_index = file_index
         self._parser = parser
+        self._parsed_seq_len = parsed_seq_len
 
         super().__init__()
 
@@ -34,10 +41,14 @@ def from_path(
         parser: TorchSequenceParser,
         file_format: str = "fasta",
         alphabet: EncodingAlphabet = ExtendedIUPACDNAEncoding(),
+        index_db: str = ":memory:",
+        parsed_seq_len: Optional[int] = None,
     ) -> "GenomicDataset":
         """Init from a single file. This is a convience method that delegates to from_paths."""
 
-        return cls.from_paths([path], parser, file_format, alphabet)
+        return cls.from_paths(
+            [path], parser, file_format, alphabet, index_db, parsed_seq_len=parsed_seq_len
+        )
 
     @classmethod
     def from_paths(
@@ -47,14 +58,15 @@ def from_paths(
         file_format: str = "fasta",
         alphabet: EncodingAlphabet = ExtendedIUPACDNAEncoding(),
         index_db: str = ":memory:",
+        parsed_seq_len: Optional[int] = None,
         **kwargs,
     ) -> "GenomicDataset":
         """Initialize the GenomicDataset from a pathlib.Path sequence."""
 
         file_index = SeqIO.index_db(
             index_db, [str(p) for p in path_sequence], file_format, alphabet=alphabet, **kwargs
         )
-        return cls(file_index, parser)
+        return cls(file_index, parser, parsed_seq_len)
 
     def __len__(self) -> int:
         """Return the length of the dataset."""
@@ -69,4 +81,4 @@ def __getitem__(self, i: int):
         file_name = Path(self._file_index._filenames[file_number])
 
         r = GCGCRecord(path=file_name, seq_record=self._file_index[key])
-        return self._parser.parse_record(r)
+        return self._parser.parse_record(r, self._parsed_seq_len)

gcgc/ml/pytorch_utils/parser.py

@@ -3,6 +3,7 @@
 """PyTorch specific parser."""
 
 from typing import Dict
+from typing import Optional
 
 import torch
 
@@ -13,11 +14,15 @@
 class TorchSequenceParser(SequenceParser):
     """A PyTorch Sequence Parser."""
 
-    def parse_record(self, gcgc_record: GCGCRecord) -> Dict:
+    def parse_record(self, gcgc_record: GCGCRecord, parsed_seq_len: Optional[int] = None) -> Dict:
         """Convert the incoming SeqRecord to a dictionary of features."""
 
-        parsed_features = super().parse_record(gcgc_record)
+        parsed_features = super().parse_record(gcgc_record, parsed_seq_len)
         parsed_features["seq_tensor"] = torch.LongTensor(parsed_features["seq_tensor"])
+        parsed_features["seq_tensor_masked"] = torch.LongTensor(
+            parsed_features["seq_tensor_masked"]
+        )
+        parsed_features["seq_len"] = torch.tensor(parsed_features["seq_len"])
 
         if self.has_offset:
             parsed_features["offset_seq_tensor"] = torch.LongTensor(