Carver and parser for torrent resume.dat entries (#2234)

iped-app/resources/config/conf/CarverConfig.xml

@@ -277,6 +277,19 @@
             <mediaType>application/x-bittorrent-resume-dat</mediaType>
         </carverType>
 
+        <carverType>
+            <name>RESUME.DAT ENTRY</name>
+            <signatures>
+                <headerSignature>d8:added_oni</headerSignature>
+                <footerSignature>8:uploadedi</footerSignature>
+            </signatures>
+            <carverClass>iped.carvers.custom.ResumeDatEntryCarver</carverClass>
+            <minLength>256</minLength>
+            <maxLength>32768</maxLength>
+            <mediaType>application/x-bittorrent-resume-dat-entry</mediaType>
+            <stopOnNextHeader>true</stopOnNextHeader>
+        </carverType>
+
         <!--carverType>
             <name>PLIST</name>
             <signatures>

iped-app/resources/config/conf/CategoriesConfig.json

@@ -108,7 +108,7 @@
       {"name": "Ares Galaxy", "mimes": ["application/x-ares-galaxy","application/x-ares-galaxy-entry"]},
       {"name": "E-Mule", "mimes": ["application/x-emule", "application/x-emule-part-met", "application/x-emule-searches", "application/x-emule-preferences-ini", "application/x-emule-preferences-dat", "application/x-emule-known-met-entry", "application/x-emule-part-met-entry"]},
       {"name": "Shareaza", "mimes": ["application/x-shareaza-searches-dat", "application/x-shareaza-library-dat", "application/x-shareaza-library-dat-entry", "application/x-shareaza-download"]},
-      {"name": "Torrent", "mimes": ["application/x-bittorrent-resume-dat","application/x-bittorrent-settings-dat", "application/x-bittorrent"]},
+      {"name": "Torrent", "mimes": ["application/x-bittorrent-resume-dat", "application/x-bittorrent-resume-dat-entry", "application/x-bittorrent-settings-dat", "application/x-bittorrent"]},
       {"name": "Other Peer-to-peer", "mimes": ["application/x-p2p"]}
     ]},
     {"name": "Browser Artifacts", "categories":[

iped-app/resources/config/conf/MakePreviewConfig.txt

@@ -10,4 +10,4 @@ supportedMimes = application/x-whatsapp-db; application/x-whatsapp-db-f; applica
 supportedMimes = application/x-prefetch; text/x-vcard; application/x-emule-preferences-dat; application/vnd.android.package-archive; application/x-bittorrent-settings-dat
 
 # List of mimetypes which parsers insert links to other case items into preview
-supportedMimesWithLinks = application/x-emule; application/x-emule-part-met; application/x-ares-galaxy; application/x-shareaza-library-dat; application/x-shareaza-download; application/x-bittorrent-resume-dat; application/x-bittorrent
+supportedMimesWithLinks = application/x-emule; application/x-emule-part-met; application/x-ares-galaxy; application/x-shareaza-library-dat; application/x-shareaza-download; application/x-bittorrent-resume-dat; application/x-bittorrent-resume-dat-entry; application/x-bittorrent

iped-app/resources/config/conf/ParserConfig.xml

@@ -351,6 +351,7 @@
             </params>
         </parser>
         <parser class="iped.parsers.bittorrent.BitTorrentResumeDatParser"></parser>
+        <parser class="iped.parsers.bittorrent.BitTorrentResumeDatEntryParser"></parser>
         <parser class="iped.parsers.bittorrent.BitTorrentGenericDatParser"></parser>
         <parser class="iped.parsers.bittorrent.TorrentFileParser"></parser>
         <parser class="iped.parsers.lnk.LNKShortcutParser"></parser>

iped-app/resources/config/profiles/triage/conf/ParserConfig.xml

@@ -335,6 +335,7 @@
         <parser class="iped.parsers.shareaza.ShareazaSearchesDatParser"></parser>
         <parser class="iped.parsers.shareaza.ShareazaLibraryDatParser"></parser>
         <parser class="iped.parsers.bittorrent.BitTorrentResumeDatParser"></parser>
+        <parser class="iped.parsers.bittorrent.BitTorrentResumeDatEntryParser"></parser>
         <parser class="iped.parsers.bittorrent.TorrentFileParser"></parser>
         <parser class="iped.parsers.lnk.LNKShortcutParser"></parser>
         <parser class="iped.parsers.misc.OFXParser"></parser>

iped-app/src/main/java/iped/app/ui/IconManager.java

@@ -194,6 +194,7 @@ private static Map<String, QualityIcon> initMimeToIconMap(int size) {
         icon = availableIconsMap.get("torrent");
         if (icon != null) {
             mimeIconMap.put("application/x-bittorrent-resume-dat", icon);
+            mimeIconMap.put("application/x-bittorrent-resume-dat-entry", icon);
             mimeIconMap.put("application/x-bittorrent-settings-dat", icon);
             mimeIconMap.put("application/x-bittorrent", icon);
         }

iped-carvers/iped-carvers-impl/src/main/java/iped/carvers/custom/ResumeDatEntryCarver.java

@@ -0,0 +1,122 @@
+package iped.carvers.custom;
+
+import java.io.IOException;
+
+import iped.carvers.api.CarverType;
+import iped.carvers.api.Hit;
+import iped.carvers.standard.AbstractCarver;
+import iped.data.IItem;
+import iped.io.SeekableInputStream;
+
+public class ResumeDatEntryCarver extends AbstractCarver {
+    private static final int maxNameBlockLen = 512;
+    private static final int minNameBlockLen = 8;
+    private static final int maxSeedtimeBlockLen = 16;
+
+    @Override
+    public boolean isSpecificIgnoreCorrupted() {
+        return true;
+    }
+
+    @Override
+    protected String getCarvedNamePrefix() {
+        return "Carved-Resume-Entry-";
+    }
+
+    @Override
+    protected String getCarvedNameSuffix() {
+        return ".dat";
+    }
+
+    @Override
+    public IItem carveFromFooter(IItem parentEvidence, Hit footer) throws IOException {
+        String parentType = parentEvidence.getMediaType().toString();
+        if (parentType.toLowerCase().contains("torrent")) {
+            return null;
+        }
+
+        Hit header = headersWaitingFooters.pollLast();
+        if (header != null) {
+
+            // Try to move the header backwards, to include the entry name (no tag)
+            long start = Math.max(0, header.getOffset() - maxNameBlockLen);
+            int read = (int) (header.getOffset() - start);
+            if (read <= minNameBlockLen) {
+                return null;
+            }
+
+            byte[] bytes = null;
+            try (SeekableInputStream is = parentEvidence.getSeekableInputStream()) {
+                is.seek(start);
+                bytes = is.readNBytes(read);
+            } catch (Exception e) {
+                return null;
+            }
+            if (bytes == null) {
+                return null;
+            }
+
+            int back = -1;
+            for (int i = bytes.length - 1; i >= 0; i--) {
+                if ((bytes[i] & 255) == ':') {
+                    int len = bytes.length - i - 1;
+                    char[] s = String.valueOf(len).toCharArray();
+                    for (int j = 0; j < s.length; j++) {
+                        if ((bytes[i - s.length + j] & 255) != s[j]) {
+                            return null;
+                        }
+                    }
+                    back = bytes.length - i + s.length;
+                    break;
+                }
+            }
+            if (back == -1) {
+                return null;
+            }
+            header = new Hit(header.getSignature(), header.getOffset() - back);
+
+            // Try to move the footer forward, to include the end of the last tag
+            bytes = null;
+            start = footer.getOffset() + footer.getSignature().getLength();
+            read = (int) Math.min(start + maxSeedtimeBlockLen, parentEvidence.getLength() - start);
+            try (SeekableInputStream is = parentEvidence.getSeekableInputStream()) {
+                is.seek(start);
+                bytes = is.readNBytes(read);
+            } catch (Exception e) {
+                return null;
+            }
+            if (bytes == null) {
+                return null;
+            }
+
+            int forward = -1;
+            for (int i = 0; i < bytes.length; i++) {
+                char c = (char) (bytes[i] & 255);
+                if (c == 'e' || c == 'i' || c == 'd') {
+                    forward = i + 1;
+                }
+                if (c == ':') {
+                    break;
+                }
+            }
+
+            if (forward == -1) {
+                return null;
+            }
+
+            long len = footer.getOffset() + footer.getSignature().getLength() - header.getOffset() + forward;
+            CarverType typeCarved = header.getSignature().getCarverType();
+            if (len < typeCarved.getMinLength() || len > typeCarved.getMaxLength()) {
+                return null;
+            }
+            IItem item = carveFromHeader(parentEvidence, header, len);
+            return item;
+        }
+        return null;
+    }
+
+    @Override
+    public long getLengthFromHit(IItem parentEvidence, Hit headerOffset) throws IOException {
+        return 0;
+    }
+}

iped-engine/src/main/java/iped/engine/core/QueuesProcessingOrder.java

@@ -10,6 +10,7 @@
 import org.apache.tika.mime.MediaTypeRegistry;
 
 import iped.parsers.ares.AresParser;
+import iped.parsers.bittorrent.BitTorrentResumeDatEntryParser;
 import iped.parsers.bittorrent.BitTorrentResumeDatParser;
 import iped.parsers.bittorrent.TorrentFileParser;
 import iped.parsers.browsers.chrome.CacheIndexParser;
@@ -81,6 +82,7 @@ private static Map<MediaType, Integer> installTypesToPostProcess() {
 
         mediaTypes.put(MediaType.parse(TorrentFileParser.TORRENT_FILE_MIME_TYPE), 2);
         mediaTypes.put(MediaType.parse(BitTorrentResumeDatParser.RESUME_DAT_MIME_TYPE), 3);
+        mediaTypes.put(MediaType.parse(BitTorrentResumeDatEntryParser.RESUME_DAT_ENTRY_MIME_TYPE), 3);
 
         mediaTypes.put(WhatsAppParser.WA_DB, 2);
         mediaTypes.put(WhatsAppParser.MSG_STORE, 3);

iped-engine/src/main/java/iped/engine/task/P2PBookmarker.java

@@ -18,6 +18,7 @@
 import iped.engine.search.IPEDSearcher;
 import iped.engine.task.index.IndexItem;
 import iped.parsers.ares.AresParser;
+import iped.parsers.bittorrent.BitTorrentResumeDatEntryParser;
 import iped.parsers.bittorrent.BitTorrentResumeDatParser;
 import iped.parsers.emule.KnownMetParser;
 import iped.parsers.emule.PartMetParser;
@@ -105,6 +106,8 @@ public void createBookmarksForSharedFiles(File caseDir) {
                 HashTask.HASH.SHA1.toString(), HashTask.HASH.EDONKEY.toString());
         p2pPrograms.put(BitTorrentResumeDatParser.RESUME_DAT_MIME_TYPE,
                 new P2PProgram(torrentHashes, "Torrent", new Color(0, 160, 60)));
+        p2pPrograms.put(BitTorrentResumeDatEntryParser.RESUME_DAT_ENTRY_MIME_TYPE,
+                new P2PProgram(torrentHashes, "Torrent", new Color(0, 160, 60)));
 
         P2PProgram progGDrive = new P2PProgram(HashTask.HASH.MD5.toString(), "GoogleDrive");
         p2pPrograms.put(GDriveCloudGraphParser.GDRIVE_CLOUD_GRAPH_REG.toString(), progGDrive);

iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/bittorrent/BitTorrentResumeDatEntryParser.java

@@ -0,0 +1,177 @@
+package iped.parsers.bittorrent;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Collections;
+import java.util.Set;
+import java.util.TimeZone;
+
+import org.apache.commons.codec.binary.Hex;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.HttpHeaders;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.parser.AbstractParser;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.sax.XHTMLContentHandler;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
+
+import iped.data.IItemReader;
+import iped.parsers.util.Messages;
+import iped.parsers.util.P2PUtil;
+import iped.properties.BasicProps;
+import iped.properties.ExtraProperties;
+import iped.search.IItemSearcher;
+import iped.utils.LocalizedFormat;
+
+public class BitTorrentResumeDatEntryParser extends AbstractParser {
+    private static final long serialVersionUID = 9008710913652882111L;
+    private static final Set<MediaType> SUPPORTED_TYPES = Collections
+            .singleton(MediaType.application("x-bittorrent-resume-dat-entry"));
+    public static final String RESUME_DAT_ENTRY_MIME_TYPE = "application/x-bittorrent-resume-dat-entry";
+
+    private static final String[] header = new String[] { 
+            Messages.getString("BitTorrentResumeDatParser.TorrentFile"),
+            Messages.getString("BitTorrentResumeDatParser.RootDir"),
+            Messages.getString("BitTorrentResumeDatParser.Path"),
+            Messages.getString("BitTorrentResumeDatParser.InfoHash"),
+            Messages.getString("BitTorrentResumeDatParser.Downloaded"),
+            Messages.getString("BitTorrentResumeDatParser.Uploaded"),
+            Messages.getString("BitTorrentResumeDatParser.AddedDate"),
+            Messages.getString("BitTorrentResumeDatParser.CompletedDate"),
+            Messages.getString("BitTorrentResumeDatParser.Time"),
+            Messages.getString("BitTorrentResumeDatParser.LastSeenCompleteDate"),
+            Messages.getString("BitTorrentResumeDatParser.SeedTime"),
+            Messages.getString("BitTorrentResumeDatParser.RunTime"),
+            Messages.getString("BitTorrentResumeDatParser.TorrentFoundInCase"),
+            Messages.getString("BitTorrentResumeDatParser.FilesFoundInCase") };
+
+    private static final String strYes = Messages.getString("BitTorrentResumeDatParser.Yes");
+
+    @Override
+    public Set<MediaType> getSupportedTypes(ParseContext context) {
+        return SUPPORTED_TYPES;
+    }
+
+    @Override
+    public void parse(InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context)
+            throws IOException, SAXException, TikaException {
+
+        final DateFormat df = new SimpleDateFormat(Messages.getString("BitTorrentResumeDatParser.DateFormat"));
+        df.setTimeZone(TimeZone.getTimeZone("GMT+0"));
+
+        metadata.set(HttpHeaders.CONTENT_TYPE, RESUME_DAT_ENTRY_MIME_TYPE);
+        metadata.remove(TikaCoreProperties.RESOURCE_NAME_KEY);
+
+        // Generally stream.readAllBytes() is not safe and should be avoided because it
+        // may cause OOME with large misdetected files. But in this specific case, just
+        // carved resume.dat entries, limited to 32KB by the carving configuration, are
+        // handled by this parser.
+        byte[] bytes = stream.readAllBytes();
+
+        char[] prev = "d1:X1:0".toCharArray();
+        byte[] in = new byte[prev.length + bytes.length];
+        for (int i = 0; i < prev.length; i++) {
+            in[i] = (byte) prev[i];
+        }
+        System.arraycopy(bytes, 0, in, prev.length, bytes.length);
+        ByteArrayInputStream is = new ByteArrayInputStream(in);
+        BencodedDict dict = new BencodedDict(is, df, true);
+
+        XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata);
+        xhtml.startDocument();
+        xhtml.startElement("style");
+        xhtml.characters(".dt {border-collapse: collapse; font-family: Arial, sans-serif; } "
+                + ".a { background-color:#AAAAEE; font-weight: bold; border: solid; border-width: thin; padding: 3px; text-align: left; vertical-align: middle; word-wrap: nowrap; } "
+                + ".b { border: solid; border-width: thin; padding: 3px; text-align: left; vertical-align: middle; word-wrap: nowrap; } "
+                + ".h { font-weight: bold; border: solid; border-width: thin; padding: 3px; text-align: left; vertical-align: middle; white-space: nowrap; font-family: monospace; }");
+        xhtml.endElement("style");
+        xhtml.newline();
+        try {
+            xhtml.startElement("table", "class", "dt");
+
+            IItemSearcher searcher = context.get(IItemSearcher.class);
+            for (String torrent : dict.keySet()) {
+                if (torrent.equals("X")) {
+                    continue;
+                }
+                BencodedDict torrentDict = dict.getDict(torrent);
+                if (torrentDict == null) {
+                    continue;
+                }
+                byte[] infoBytes = torrentDict.getBytes("info");
+                String infoHash = "";
+                if (infoBytes != null) {
+                    infoHash = Hex.encodeHexString(infoBytes, false);
+                }
+                int filesFoundInCase = 0;
+                IItemReader item = P2PUtil.searchItemInCase(searcher, TorrentFileParser.TORRENT_INFO_HASH, infoHash);
+                if (item != null) {
+                    metadata.add(ExtraProperties.LINKED_ITEMS, BasicProps.HASH + ":" + item.getHash());
+                    String[] values = item.getMetadata().getValues(ExtraProperties.LINKED_ITEMS);
+                    if (values != null) {
+                        Long uploaded = torrentDict.getLong("uploaded");
+                        boolean isShared = uploaded != null && uploaded > 0;
+                        for (String v : values) {
+                            metadata.add(ExtraProperties.LINKED_ITEMS, v);
+                            if (isShared) {
+                                int p = v.lastIndexOf(':');
+                                if (p >= 0) {
+                                    v = v.substring(p + 1).trim();
+                                }
+                                metadata.add(ExtraProperties.SHARED_HASHES, v);
+                            }
+                        }
+                    }
+                    String v = item.getMetadata().get(TorrentFileParser.TORRENT_FILES_FOUND_IN_CASE);
+                    if (v != null && !v.isBlank()) {
+                        filesFoundInCase = Integer.parseInt(v);
+                    }
+                }
+
+                String[] rowElements = new String[] { 
+                        torrent, 
+                        torrentDict.getString("rootdir"),
+                        torrentDict.getString("path"), 
+                        infoHash,
+                        LocalizedFormat.format(torrentDict.getLong("downloaded")),
+                        LocalizedFormat.format(torrentDict.getLong("uploaded")), 
+                        torrentDict.getDate("added_on"),
+                        torrentDict.getDate("completed_on"), 
+                        torrentDict.getDate("time"),
+                        torrentDict.getDate("last seen complete"),
+                        LocalizedFormat.format(torrentDict.getLong("seedtime")),
+                        LocalizedFormat.format(torrentDict.getLong("runtime")), 
+                        item != null ? strYes : "",
+                        filesFoundInCase > 0 ? String.valueOf(filesFoundInCase) : "" };
+
+                for (int i = 0; i < rowElements.length; i++) {
+                    String c = rowElements[i];
+                    if (!c.isBlank()) {
+                        xhtml.startElement("tr");
+
+                        xhtml.startElement("td", "class", "a");
+                        xhtml.characters(header[i]);
+                        xhtml.endElement("td");
+
+                        xhtml.startElement("td", "class", i == 3 ? "h" : "b");
+                        xhtml.characters(c);
+                        xhtml.endElement("td");
+
+                        xhtml.endElement("tr");
+                    }
+                }
+                break;
+            }
+
+            xhtml.endElement("table");
+        } finally {
+            xhtml.endDocument();
+        }
+    }
+}