--- /dev/null
+package goby;
+
+option java_package = "edu.cornell.med.icb.goby.alignments";
+
+option optimize_for = SPEED;
+
+/*
+ This message is written to 'basename'.entries as a very large chunked collection.
+*/
+message AlignmentCollection {
+ repeated AlignmentEntry alignment_entries = 1;
+}
+
+
+message AlignmentEntry {
+ /* Multiplicity of this entry. The number of times this alignment entry would be repeated exactly the same if
+ query redundancy had not been removed by read factorization.
+ */
+ optional uint32 multiplicity = 7;
+
+ /*
+ Compressed stream of data. Removed since Goby 2.0 supports chunk codecs. Do not reuse field index 23
+ optional bytes compressed_data = 23;
+ */
+
+ /* An integer that uniquely identifies the query (a short read) in a set of alignment runs. When several
+ alignment runs are made with the same set of query sequences, equality of query index means that the query
+ sequences were the same. (Comparing integers for equality is much faster than comparing strings.)
+ This field is required (enforced by semantic validation in Goby 2.0+).
+ */
+ optional uint32 query_index = 1;
+ /* An integer that uniquely identifies the target (e.g., a chromosome) in a set of alignment runs. When several
+ alignment runs are made with the same set of target sequences, equality of target index means that the target
+ sequence was the same across the runs. (Comparing integers for equality is much faster than comparing strings.)
+ This field is required (enforced by semantic validation in Goby 2.0+).
+ */
+ optional uint32 target_index = 2;
+ /*
+ The position on the target of the start of the alignment between the query and the target.
+ In the following example, position is 3 because the third base of the query 'C' was aligned with
+ position 3 of the reference (two read bases were soft clipped: "ct"). This example shows that the
+ alignment can start at a mismatch if it was so constructed by the aligner.
+
+ 0123456789
+ AAAAGTCAAA target
+ ctCGTC query
+ This field is required (enforced by semantic validation in Goby 2.0+).
+ */
+ optional uint32 position = 3;
+
+ /*
+ True when the query matches the target on the reverse strand
+ */
+ optional bool matching_reverse_strand = 6;
+
+ /*
+ The position on the query where the alignment starts. This value is different from zero
+ when some bases/residues of the query could not be aligned with the target.
+ TODO: Rename this to left_trim. Add a right_trim property.
+ */
+ optional uint32 query_position = 5;
+
+ /*
+ The score of the alignment, where larger scores indicate better matches between the query and the target.
+ If an aligner outputs only the number of mismatches between query and target, the score is taken to be
+ -(#mismatches(query,target)).
+ */
+ optional float score = 4;
+
+ /*
+ Number of bases/residues that differ in the alignment between query and target sequences.
+ */
+ optional uint32 number_of_mismatches = 8;
+
+ /*
+ Cumulative number of insertions and/or deletions present in the alignment.
+ */
+ optional uint32 number_of_indels = 9;
+
+ /*
+ Number of bases that have been aligned for the query. Please note that query_aligned_length must be
+ less or equal to query_length.
+ */
+ optional uint32 query_aligned_length = 11;
+
+ /*
+ Number of bases that have been aligned for the target.
+ */
+ optional uint32 target_aligned_length = 12;
+
+ repeated SequenceVariation sequence_variations = 13;
+
+ /*
+ Length of the query sequence.
+ */
+ optional uint32 query_length = 10;
+ /*
+ Mapping Quality (phred-scaled posterior probability that the mapping
+ position of this read is incorrect). Please note that different aligners
+ may estimate mapping quality with different approaches, resulting in aligner
+ specific differences in the distribution of mapping quality. It is recommended
+ to condition mapping quality on the aligner that produced the specific alignment
+ being processed. See aligner name and version in the header.
+ Note that the following description is preliminary. A clear specification is
+ needed:
+ The mapping quality should be proportional to the
+ log of the probability that the given mapping is the "correct" one.
+ So if there are five equally good mappings of a read to the genome,
+ the probability of each would be 0.2, and the mapping quality would be
+ something like -10*log10(1-0.2) = 1. If a mapping is highly likely,
+ say a 1e-4 of it being wrong, then the mapping quality would be
+ -10*log10(1e-4) = 40.
+ */
+ optional int32 mapping_quality = 14;
+
+ /*
+ If this read was aligned with a pair, the flags for the pair alignment (based on SAM):
+ 000000001 paired
+ 000000010 properly paired
+ 000000100 read unmapped
+ 000001000 mate unmapped
+ 000010000 read reverse strand
+ 000100000 mate reverse strand
+ 001000000 first in pair
+ 010000000 second in pair
+ 100000000 not primary alignment
+ */
+ optional uint32 pair_flags = 15;
+
+ /*
+ If there is an alignment entry for the paired read (the paired read was mapped), a link to the entry is given.
+ */
+ optional RelatedAlignmentEntry pair_alignment_link = 16;
+
+ /* Index of the read fragment from which this alignment was obtained. */
+ optional uint32 fragment_index = 17;
+
+ /* If a read spans exon-exon junctions some aligners (e.g., GSNAP) will output two or more
+ alignment entries, one for each matching part of the read, and link these entries with
+ spliced_alignment_links. The field spliced_forward_alignment_link points to the next
+ AlignmentEntry in the chain of spliced alignments.
+ */
+ optional RelatedAlignmentEntry spliced_forward_alignment_link = 18;
+
+ /* If a read spans exon-exon junctions some aligners (e.g., GSNAP) will output two or more
+ alignment entries, one for each matching part of the read, and link these entries with
+ spliced_alignment_links. The field spliced_backward_alignment_link points to the previous
+ AlignmentEntry in the chain of spliced alignments.
+ */
+ optional RelatedAlignmentEntry spliced_backward_alignment_link = 22;
+
+ /*
+ If a read spans exon-exon junctions some aligners (e.g., GSNAP) will output two alignment entries, one for each
+ matching part of the read, and flag describes the spliced_alignment_link with these
+ binary flags:
+ 000000001 normal
+ 000000010 novel
+ */
+ optional uint32 spliced_flags = 19;
+
+ /* The size of the insert used when making the sequence library. This is the total size of the DNA
+ fragment to sequence, without the adapters. This is not the length of sequence that separates the reads.
+ See http://seqanswers.com/forums/showthread.php?t=8730 for details. Insert size is inferred for each pair
+ of reads by the aligner and is recorded here if was estimated (i.e., for paired-end reads).
+ */
+ optional sint32 insert_size = 20;
+
+ /*
+ The sample index. Uniquely identifies the aligned sample this read was read from. Storing the sample index in the
+ alignment entry makes it possible to concat alignments from different origins and track what sample originally
+ contained each entry.
+ */
+ optional uint32 sample_index = 21;
+ /*
+ The total number of times the query index associated with this entry occurs across the entire alignment file.
+
+ This field is used to purge queryIndex->smallIndex associations after all instances of a queryindex have
+ been seen (see QueryIndexPermutation class). When each entry has a value for this field, the header field
+ query_index_occurrences is true.
+ This field is required (enforced by semantic validation in Goby 2.0+).
+ */
+ optional uint32 query_index_occurrences = 25;
+ /*
+ The total number of times the read matches the reference across the entire alignment file. This differs from
+ query_index_occurrences because reads that are matching through splice and pair links count as one for ambiguity.
+ The field can be used to filter by ambiguity-threshold on the fly after an alignment has been done (to restrict
+ entries to more smaller thresholds). When each entry has a value for this field, the header field
+ ambiguity_stored_in_entries is true.
+
+ This field is required (enforced by semantic validation in Goby 2.0+).
+ */
+ optional uint32 ambiguity = 27;
+ /*
+ List of BAM attributes, if the alignment was imported from BAM. The attributes are stored in exactly the format
+ allowed for BAM. For instance, X0:i:9 X1:i:1 MD:Z:68 RG:Z:SRR084825 will be stored as four strings:
+ "X0:i:9", "X1:i:1", "MD:Z:68", "RG:Z:SRR084825". Note that sam-to-compact will interpret some BAM attributes
+ and populate goby native fields. Such tags do not appear in bam_attributes, and are instead re-generated from
+ the corresponding goby native fields.
+ Since Goby 2.0.
+ */
+ repeated string bam_attributes = 50;
+ /*
+ Quality scores for all bases of the read.
+ Since Goby 2.0.
+ */
+ optional bytes read_quality_scores = 55;
+
+ /*
+ Origin index. An integer that references a ReadOriginInfo message in the alignment header and
+ makes it possible to track the origin of the read (especially useful after several alignments
+ have been merged/concatenated).
+ (Since Goby 2.0).
+ */
+ optional uint32 read_origin_index = 26;
+ /*
+ Bases that an aligner considered do not belong to the alignment of the read to the reference. Potentially
+ erroneous bases, or bases that belong to a different part of the reference genome. Left clipped bases are
+ stored in this field as character bases, or as an equal sign character '=' when the clipped base did match
+ the reference base. For instance "A=G" for three soft-clipped bases, the middle one matching the genome at
+ this position. The number of bases in softClippedBasesLeft is exactly equal to queryPosition.
+ */
+ optional string softClippedBasesLeft = 30;
+ /*
+ Bases that an aligner considered do not belong to the alignment of the read to the reference. Potentially
+ erroneous bases, or bases that belong to a different part of the reference genome. Right clipped bases are
+ stored in this field as character bases, or as an equal sign character '=' when the clipped base did match
+ the reference base. The number of bases in softClippedBasesRight is exactly equal
+ to queryLength - queryAlignedLength - queryPosition.
+ */
+ optional string softClippedBasesRight = 31;
+
+ /*
+ Quality scores for bases in softClippedBasesLeft. Stored in Phred Units.
+ */
+ optional bytes softClippedQualityLeft = 32;
+ /*
+ Quality scores for bases in softClippedBasesRight. Stored in Phred Units.
+ */
+ optional bytes softClippedQualityRight = 33;
+ /*
+ Sequence for a read placed near this entry, but unmapped to the reference sequence. For instance, used to record
+ the sequence of a mate that did not map to the reference. We know that the mate maps in the proximity of this entry
+ (it is placed) but are unable to map it to a specific genomic position. The sequence is always given as obtained
+ from the reads file.
+ */
+ optional string placedUnmappedSequence=40;
+ /*
+ Quality scores for a read placed near this entry. Phred units.
+ */
+ optional bytes placedUnmappedQuality=41;
+
+ /*
+ Read name. In SAM/BAM this is referred to as QNAME. Paired and segmented reads will have the same Read name.
+ */
+ optional string readName=42;
+}
+
+/* A link to another alignment entry. This message type is used to represent relations
+ between alignments, such as the relation between the two read fragments in a paired-end protocol,
+ or the relation between parts of reads that align through an exon exon junction and map in
+ different locations of the genome.
+ */
+message RelatedAlignmentEntry {
+ /* Target index of the location where the other alignment entry is mapped.
+ This field is required (enforced by semantic validation in Goby 2.0+).
+ */
+ optional uint32 target_index = 1;
+
+ /* Position on the reference where the other alignment entry is mapped. *
+ This field is required (enforced by semantic validation in Goby 2.0+).
+ */
+ optional uint32 position = 2;
+
+ /* Index of the fragment for the related alignment entry. This index
+ makes it possible to identify which of the read fragments mapped to the given
+ location is related to the source alignment entry.
+ This field is required (enforced by semantic validation in Goby 2.0+).
+ */
+ optional uint32 fragment_index = 3;
+
+ optional uint32 optimized_index=50;
+}
+
+/*
+ Represents sequence variations between the query and the reference sequences. Many variations can be represented.
+ For instance, an insertion at position 5 in the reference would be represented as from="A", to="" position=5.
+ A mutation T->G at position 6 would be rendered as from="T", to="G" position=6. Padded alignments (see SAM description)
+ can be described by a combination of pair-wise alignments, where the gap character '-' is used to indicate that no
+ base exists in the sequence considered for the alignment position, for instance:
+
+ - Padding example:
+
+ 123 (<-positions)
+ref A-C
+ A-T [from="-" to="" position=2] [from="C" to="T" position=3]
+ ACT [from="" to="C" position=2] [from="C" to="T" position=3]
+ A-T [from="-" to="" position=2] [from="C" to="T" position=3]
+
+ - Mutation example:
+ 123 (<-positions)
+ref ATT
+ ACT [from="T" to="C" position=2]
+
+ -- Example of deletion in a read:
+ 123 (<-positions)
+ref ATT
+ A-T [from="T" to="-" position=2]
+
+ -- Example of insertion of two base pairs in a read:
+ 12345 (<-positions)
+ref A--TT
+ ACCTT [from="" to="CC" position=2]
+
+ */
+message SequenceVariation {
+ /* The reference bases. Can include one or more gap characters '-', to indicate that the reference sequence has
+ no base at this alignment position.
+ This field is required (enforced by semantic validation in Goby 2.0+).
+ */
+ optional string from = 2;
+ /* The read bases that differ from the reference sequence. Can include one or more gap characters '-', to indicate
+ that the query sequence has no base at this alignment position.
+ This field is required (enforced by semantic validation in Goby 2.0+).
+ */
+ optional string to = 1;
+ /*
+ The position of the variation on the read, as if the read always matched on the forward strand.
+ Adding position to the index where the reference starts aligning the read yields the position of the variation
+ in reference/target sequence space. Since position starts at one the resulting position will also be one based.
+ This field is required (enforced by semantic validation in Goby 2.0+).
+ */
+ optional uint32 position = 3;
+ /*
+ The position of the variation, starting from the beginning of the aligned read (position 1), and up to the length
+ of the read (inclusive). Use this index if you need to know how far the variation is observed from the beginning
+ of the sequenced read. When the read has an insertion, this index records the position immediately before the base
+ where the bases are inserted (these bases are in the to field).
+ When the read has a deletion, read_index records the position in the read after which the bases that would align
+ in the reference are missing (these bases are in the from field).
+ This field is required (enforced by semantic validation in Goby 2.0+).
+ */
+ optional uint32 read_index = 5;
+
+ /**
+ The read base quality scores for those bases that are given in the to field. This field
+ is populated when the reads used to perform the search include quality scores, and when
+ the alignment parser can extract the information from the aligner's output.
+ (this option is currently not implemented in Goby.)
+ */
+ optional bytes to_quality = 4;
+
+}
+/*
+ This message is written to 'basename'.header
+*/
+
+message AlignmentHeader {
+ /*
+ The smallest possible query index in this alignment. Data stored as an array where
+ queryIndex is the array index will be stored with only the elements in the inclusive
+ range [smallestSplitQueryIndex largestSplitQueryIndex]
+ Such data structures include queryLength and some arrays in the TooManyHits data
+ structure.
+ */
+ optional uint32 smallest_split_query_index = 9;
+ /*
+ The largest possible query index in this alignment. Data stored as an array where
+ queryIndex is the array index will be stored with only the elements in the inclusive
+ range [smallestSplitQueryIndex largestSplitQueryIndex]
+ Such data structures include queryLength and some arrays in the TooManyHits data
+ structure.
+ */
+ optional uint32 largest_split_query_index = 11;
+
+ /* Mapping from query identifier name to query index (as used in alignment entries).
+ */
+ optional IdentifierMapping query_name_mapping = 1;
+
+ /* Mapping from target identifier name to target index (as used in alignment entries).
+ */
+ optional IdentifierMapping target_name_mapping = 2;
+
+ /*
+ The number of query sequences
+ */
+ optional uint32 number_of_queries = 5;
+ /*
+ The number of target sequences
+ */
+ optional uint32 number_of_targets = 6;
+ /*
+ The number of reads that were aligned to the reference and are represented in this alignment archive.
+ */
+ optional uint32 number_of_aligned_reads = 7;
+
+ /*
+ Length of the query sequences. One number per query, in the order of increasing query index.
+ This information has been moved to the individual alignment entries.
+ */
+ repeated uint32 query_length = 3 [deprecated = true];
+ /*
+ If query length is constant across all the queries, this field contains the constant length.
+ In such cases, query_length will be empty.
+ */
+ optional uint32 constant_query_length = 10;
+
+ /*
+ Length of the target sequences. One number per target, in the order of increasing target index.
+ The target indexes must be 0..(number of targets - 1).
+ */
+ repeated uint32 target_length = 8;
+ /*
+ Indicates whether this alignment is sorted by position. True: the alignment entries occur in sorted
+ order, such that entry a occurs before entry b if a.targetIndex< b.targetIndex or, when entries
+ have the same target, when a.position < b.position.
+ */
+ optional bool sorted = 13;
+
+ /*
+ Indicates whether this alignment is indexed by position. When this attribute is true, a file called
+ 'basename'.index exists that contains the AlignmentIndex message (GZip compressed).
+ */
+ optional bool indexed = 14;
+ /*
+ True when query lengths are stored in alignment entries (Goby 1.7+).
+ */
+ optional bool query_lengths_stored_in_entries = 15;
+ /*
+ Name of the aligner that produced this alignment.
+ */
+ optional string aligner_name = 17;
+ /*
+ Version number for the aligner implementation that produced this alignment.
+ */
+ optional string aligner_version = 18;
+ /*
+ The version of Goby that created this alignment file.
+ */
+ optional string version = 25;
+
+ /*
+ Sample basenames, in the order of increasing sampleIndex, starting with sampleIndex=0.
+ */
+
+ repeated string sample_basename = 30;
+
+ /*
+ This field is true when the query indices of alignment entries were permuted to smaller indices. Only sorted
+ alignments can have query_indices_were_permuted=true. When the field is true, and you need to retrieve the
+ original query-index of an alignment (because you want to retrieve the specific read(s) from a read file for
+ instance), you will need the information in the permutation file (extension basename.perm) and transform back
+ each small index of interest to the original query index.
+ */
+ optional bool query_indices_were_permuted = 26;
+ /*
+ This field is true when entries in the alignment .entries file all have the query_index_occurrences field populated
+ (Since Goby 2.0).
+ */
+ optional bool query_index_occurrences = 35;
+
+ /*
+ This field is true when entries in the alignment .entries file all have the ambiguity field populated
+ (Since Goby 2.0).
+ */
+ optional bool ambiguity_stored_in_entries = 36;
+ /*
+ This field is true when entries in the alignment .entries file all have the read_quality_score field populated.
+ (Since Goby 2.0).
+ */
+ optional bool all_read_quality_scores = 40;
+ /*
+ A description of the origin of sets of reads. Serves a similar function to BAM read groups, but more flexible and
+ efficient. Instead of storing strings, we use integers in the entries.
+ Alignemnt entries will link to a specific ReadOriginInfo with the origin_index field.
+ (Since Goby 2.0).
+ */
+ repeated ReadOriginInfo read_origin = 27;
+}
+
+message IdentifierMapping {
+ repeated IdentifierInfo mappings = 1;
+}
+
+message IdentifierInfo {
+ required string name = 1;
+ required uint32 index = 2;
+}
+
+
+/*
+ A description of the origin of sets of reads. Stored in the Goby alignment header and linked
+ from alignment entries. Goby makes it possible to adapt origin equivalence rules on the fly
+ efficiently. To do this, it is sufficient to read the header of the alignment, decide which
+ ReadOriginInfo instances are equivalent (e.g., by looking at sample, platform, library, or
+ other fields in the message), then construct a function e(a):int. This function takes
+ one originIndex parameter and returns another integer that maps to an equivalent class. The
+ equivalence class can be used to estimate error models for entries that belong to each class,
+ for instance.
+ (Since Goby 2.0).
+ */
+message ReadOriginInfo {
+ /*
+ Origin index. An integer that links alignment entries to their origin information.
+ */
+ required uint32 origin_index = 1;
+ /*
+ Identifier that describes the origin of the reads. This field is compatible with the ID/platform field of BAM read
+ groups. Free text.
+ */
+ required string origin_id = 2;
+ /*
+ The sample from which the reads were sequenced. This field is compatible with the SM/sample field of BAM read
+ groups. Free text.
+ */
+ optional string sample = 4;
+ /*
+ The platform on which the reads were sequenced. This field is compatible with the PL/platform field of BAM read
+ groups. Valid values: ILLUMINA, SOLID, LS454, HELICOS and PACBIO.
+ */
+ optional string platform = 5;
+ /*
+ The library from which the reads were sequenced. This field is compatible with the LB/library field of BAM read
+ groups. Free text.
+ */
+ optional string library = 8;
+ /*
+ The platform unit on which the reads were sequenced. This field for compatibility with samtools.
+ */
+ optional string platform_unit = 12;
+ /*
+ The date the reads were sequenced. Useful to identify batch effects, in the format dd:MMM:yyyy.
+ The month is Jan, Feb, etc. to avoid all confusion with days when day<=12.
+ */
+ optional string run_date = 6;
+}
+
+/*
+ This message is written to 'basename'.tmh
+*/
+
+message AlignmentTooManyHits {
+ /*
+ The threshold used by the aligner to determine that a query is ambiguous and should be dropped.
+ Referred to as parameter k below.
+ */
+ required uint32 aligner_threshold = 2;
+ /*
+ The hits that are assigned to several (>k) reference location.
+ */
+ repeated AmbiguousLocation hits = 1;
+
+}
+
+message AmbiguousLocation {
+ /*
+ The index of the query that matched too many times.
+ */
+ required uint32 query_index = 1;
+ /*
+ The number of hits that triggered membership in the too many hits list. The query may hit more
+ locations than reported here, since some alignment tools will just drop queries that match above
+ a threshold and stop counting. This number can be >=k.
+ */
+ required uint32 at_least_number_of_hits = 2;
+ /**
+The length of the part of the query sequence that could be matched to the target (also called depth).
+May be less than the length of the query sequence, in which case the match was not perfect. When merging
+alignments produced by searching different reference sequences, consider only at_least_number_of_hits
+from alignments that have exactly the longer depth for the query. */
+ optional uint32 length_of_match = 3;
+}
+
+/*
+ This message is written to 'basename'.index
+ */
+message AlignmentIndex {
+ /*
+ Stores one element by target sequence. Each element is the cumulative target length for the target
+ stored at index i. Assume there are four target sequences, with lengths {10, 12, 15, 34}. The field
+ targetPositionOffsets will contain: {0,10,22,37}. Such offsets can be used to calculate the absolute
+ position of a genomic location. Given targetIndex and positionOnReference, the absolute location
+ is defined as targetPositionOffsets[targetIndex]+positionOnReference.
+ */
+ repeated uint32 target_position_offsets = 1 [packed = true];
+ /*
+ The byte offsets into the compressed entries file. Byte offsets are matched with absolute position
+ by index. There should be as many elements in offsets as there are in absolutePosition
+ where chunks start which represent entries whose absolute positions are less than
+ */
+ repeated uint64 offsets = 2 [packed = true];
+ /*
+ The absolute positions of the first entry in the chunk that immediately start at offset. One element
+ per chunk in the 'basename'.entries file.
+ */
+ repeated uint64 absolute_positions = 3 [packed = true];
+
+}
projects / gentoo.git / commitdiff