dvdkhl / avdump3 Goto Github PK

View Code? Open in Web Editor NEW

28.0 6.0 8.0 49.25 MB

Metadata Extraction Tool for Media Files

License: MIT License

C# 60.12% C 39.63% Makefile 0.08% Dockerfile 0.03% Shell 0.13%

metadata hashing reporting module-system audio video cross-platform

avdump3's People

Stargazers

Watchers

Forkers

grumpybear57 zeromind astiob fvangen acidburn0zzz da3dsoul revam alysson-souza

avdump3's Issues

TTH hashes are wrong

TTH hashes are always wrong, independent of file size.

0-valued <date /> node in some .mkv dumps

Example file: https://anidb.net/f1840938

The issue seems to already exist in AVD2 and has been ported to AVD3. But not all files are affected.

   <size>84390127</size>
   <avmf>
     <date>0</date>
     <duration>210.071</duration>

The MediaInfoXMLReport for the file above gives the following:

     <Encoded_Date Unit="">UTC 2010-02-22 21:41:29</Encoded_Date>
     <File_Created_Date Unit="">UTC 2020-05-17 21:24:21.355</File_Created_Date>
     <File_Created_Date_Local Unit="">2020-05-17 23:24:21.355</File_Created_Date_Local>
     <File_Modified_Date Unit="">UTC 2020-05-17 21:24:57.600</File_Modified_Date>
     <File_Modified_Date_Local Unit="">2020-05-17 23:24:57.600</File_Modified_Date_Local>

Allow --CRC32Error to also take a full path as first argument

Currently --CRC32Error only allows a filename ("crcerr.txt") or at most a partial file path ("..\crcerr.txt") as the first argument.

Any path containing a ":" (like a drive letter) will break it.

XML spam

it looks like AVD3 is spamming my user die with XML files filled with this.

AVD3AniDBModule.ACreqing.ACReqException: Cannot query ACReq, ACreqQueue is or is being shut down at AVD3AniDBModule.ACreqing.ACreqQueue.QueryACReq(Byte[] acreqData, Object tag) in D:\Projects\C#\AVD3AniDB\AVD3AniDBModule\ACreqing\ACreqQueue.cs:line 130 at AVD3AniDBModule.AVD3AniDBModule.FileProcessed(Object sender, AVD3CLFileProcessedEventArgs e) in D:\Projects\C#\AVD3AniDB\AVD3AniDBModule\AVD3AniDBModule.cs:line 338 with Status: Shutdown

avd3 xml spam.zip

Exception "Cannot read block when EOS is reached" is sometimes thrown for small files

Unique Tracks being merged

MediaInfoLib reports a subtitle Track with an id x**-**y.
Currently only integer values are used as an id.
Previously only the first number x was used, now "-" is replaced by "0"

To properly handle this issue, I'll have to change ids from integer to string.

AVD3 XML spam

it looks like AVD3 is spamming my user die with XML files filled with this.

AVD3AniDBModule.ACreqing.ACReqException: Cannot query ACReq, ACreqQueue is or is being shut down
   at AVD3AniDBModule.ACreqing.ACreqQueue.QueryACReq(Byte[] acreqData, Object tag) in D:\Projects\C#\AVD3AniDB\AVD3AniDBModule\ACreqing\ACreqQueue.cs:line 130
   at AVD3AniDBModule.AVD3AniDBModule.FileProcessed(Object sender, AVD3CLFileProcessedEventArgs e) in D:\Projects\C#\AVD3AniDB\AVD3AniDBModule\AVD3AniDBModule.cs:line 338 with Status: Shutdown

avd3 xml spam.zip

Divide by Zero Exception

{"type":1,"target":"AVDump:Message","arguments":[{"filePath":"/shoko/(Hi10)_Bleach_-_Sennen_Kessen-hen_-_14_(1080p)_(Lazy)_(14D12867).mkv","videoID":1,"commandID":4,"type":"Ended","progress":100.0,"success":false,"message":"Testing anidb connection\nAccepted files: 1\n\n","errorMessage":"Unhandled exception. System.DivideByZeroException: Attempted to divide by zero.\nat AVDump3CL.AVD3Console.OnWriteProgress(Object _)\nat System.Threading.TimerQueueTimer.<>c.<.cctor>b__27_0(Object state)\nat System.Threading.ExecutionContext.RunInternal(ExecutionContext executionContext, ContextCallback callback, Object state)\n--- End of stack trace from previous location ---\nat System.Threading.ExecutionContext.RunInternal(ExecutionContext executionContext, ContextCallback callback, Object state)\nat System.Threading.TimerQueueTimer.CallCallback(Boolean isThreadPool)\nat System.Threading.TimerQueueTimer.Fire(Boolean isThreadPool)\nat System.Threading.TimerQueue.FireNextTimers()\nat System.Threading.TimerQueue.AppDomainTimerCallback(Int32 id)\n","startedAt":"2023-07-30T18:17:17.6700343Z","endedAt":"2023-07-30T18:17:23.2006602Z"}]}�

AVDump3/AVDump3CL/AVD3ProgressDisplay.cs

Lines 42 to 97 in d96bce1

 public void WriteProgress(AVD3ConsoleProgressBuilder pb) { 

 if(state == 0) { 

 prevP = curP; 

 curP = getProgress(); 

 } 

 var interpolationFactor = (state + 1) / (double)UpdatePeriodInTicks; 

 state++; 

 state %= UpdatePeriodInTicks; 

 var processedMiBsInInterval = ((curP.BytesProcessed - prevP.BytesProcessed) >> 20) / UpdatePeriodInTicks; 

 if((DateTimeOffset.UtcNow - curP.StartedOn).TotalMinutes < 1) { 

 var now = DateTimeOffset.UtcNow; 

 var prevSpeed = (prevP.BytesProcessed >> 20) / (now - prevP.StartedOn).TotalSeconds; 

 var curSpeed = (curP.BytesProcessed >> 20) / (now - curP.StartedOn).TotalSeconds; 

 totalSpeedAverages[0] = (float)(prevSpeed + interpolationFactor * (curSpeed - prevSpeed)); 

 } else { 

 var maInterval = MeanAverageMinuteInterval; 

 totalSpeedAverages[0] = totalSpeedAverages[0] * (maInterval - 1) / maInterval + (processedMiBsInInterval / (float)AVD3Console.TickPeriod * 1000) / maInterval; 

 } 

 if((DateTimeOffset.UtcNow - curP.StartedOn).TotalMinutes < 5) { 

 totalSpeedAverages[1] = totalSpeedAverages[0]; 

 } else { 

 var maInterval = MeanAverageMinuteInterval * 5; 

 totalSpeedAverages[1] = totalSpeedAverages[1] * (maInterval - 1) / maInterval + (processedMiBsInInterval / (float)AVD3Console.TickPeriod * 1000) / maInterval; 

 } 

 if((DateTimeOffset.UtcNow - curP.StartedOn).TotalMinutes < 15) { 

 totalSpeedAverages[2] = totalSpeedAverages[1]; 

 } else { 

 var maInterval = MeanAverageMinuteInterval * 15; 

 totalSpeedAverages[2] = totalSpeedAverages[2] * (maInterval - 1) / maInterval + (processedMiBsInInterval / (float)AVD3Console.TickPeriod * 1000) / maInterval; 

 } 

 if(totalSpeedAverages[0] > 9999999 || totalSpeedAverages[1] > 9999999 || totalSpeedAverages[2] > 9999999) { 

 totalSpeedDisplayAverages[0] = (int)totalSpeedAverages[0] >> 20; 

 totalSpeedDisplayAverages[1] = (int)totalSpeedAverages[1] >> 20; 

 totalSpeedDisplayAverages[2] = (int)totalSpeedAverages[2] >> 20; 

 totalSpeedDisplayUnit = "TiB/s"; 

 } else if(totalSpeedAverages[0] > 9999 || totalSpeedAverages[1] > 9999 || totalSpeedAverages[2] > 9999) { 

 totalSpeedDisplayAverages[0] = (int)totalSpeedAverages[0] >> 10; 

 totalSpeedDisplayAverages[1] = (int)totalSpeedAverages[1] >> 10; 

 totalSpeedDisplayAverages[2] = (int)totalSpeedAverages[2] >> 10; 

 totalSpeedDisplayUnit = "GiB/s"; 

 } else { 

 totalSpeedDisplayAverages[0] = (int)totalSpeedAverages[0]; 

 totalSpeedDisplayAverages[1] = (int)totalSpeedAverages[1]; 

 totalSpeedDisplayAverages[2] = (int)totalSpeedAverages[2]; 

 totalSpeedDisplayUnit = "MiB/s"; 

 } 

 Display(pb, interpolationFactor); 

 pb.SpecialJitterEvent = state == 0; 

 }

Hashes are wrong for big files

CRC32, Ed2k, MD5, SHA1, SHA256, SHA384 and SHA512 hashes are wrong for files 2GiB or larger but correct for 2GiB-1byte files.

TTH hashes are also wrong, but they are also wrong for smaller files (issue #4).

Haven’t tested the other hash consumers.

AVD3 eats cursors

AVD3 (like AVD2) switches off the terminal cursor. Killing it with Ctrl-C leaves you with no cursor. Can also happen when it crashes.
Happens on both Windows and Linux.

Reimplement --version parameter

Reference output:

AVDump2CL.exe --version
1.0.7101.0
MediaInfoLib - v18.12

Reimplement --PrintEd2kLink parameter

Implement the parameter to output a hashed file's ed2k link again.

Exception while parsing an ogm file (Argument out of range)

4hdsWzNr.txt

When signaling the process to exit (Ctrl+C) an exception is occasionally thrown

'IOException: Cannot access file' when conccurently proccessing files

AVD Version: 8188 (From AniDB wiki download link)
Command: .\AVDump3CL.exe --Concurrent=4 --Ed2kLogPath={snip path}\ed2k.txt {snipped other args} {snipped path to files}
Exception: Error System.IO.IOException: The process cannot access the file '{snip path}\ed2k.txt' because it is being used by another process.

This would probably be hard to reproduce intentionally, so I can't be 100% sure what caused it. My suspicion is, when processing multiple files concurrently, if two or more files finish at the same time, the ed2k hash file writer is called without any safety to ensure that the file isn't already open. I can also see that I'm missing two ed2k hashes from the file from either when or after the exception was thrown (don't know what file it threw on).

The process cannot access the file XYZ because it is being used by another process.

https://anidb.net/admin/avmf/error/?do=xml&id=723792
https://anidb.net/admin/avmf/error/?do=xml&id=723953
https://anidb.net/admin/avmf/error/?do=xml&id=723952
https://anidb.net/admin/avmf/error/?do=xml&id=723951
https://anidb.net/admin/avmf/error/?do=xml&id=723785

[23:50:22] <&worf> are you sometimes running multiple instances of avd3 in parallel, Soulweaver?
[23:50:40] <Soulweaver> just one
[23:52:01] <Soulweaver> I do have --Concurrent=4 in my arguments though

These errors are local errors and really not all that important for us. Is there a way to log these kinds of errors just locally but not upload them to the server?
Alternatively, fix the root cause? :p

Tiger & TTH Hashes wrong

AVD3 8213 calculates wrong ed2k hashes for files <9728000 bytes

AVD2 7101: ed2k://|file|test_ext.txt|19837|B457600C8C6299E92C44BA96A8F39C81|/
AVD3 8213: ed2k://|file|test_ext.txt|19837|5FF8FBF6383EC38C518D31A6195689E8|/

AVD2 7101: ed2k://|file|Kyoshin to Hyouka no Shiro [Web] (Episode 002) [F2B295EC] [A-R].mkv|10014562|C3299C512ED1923F74EE49232A03C9EE|/
AVD3 8213: ed2k://|file|Kyoshin to Hyouka no Shiro [Web] (Episode 002) [F2B295EC] [A-R].mkv|10014562|C3299C512ED1923F74EE49232A03C9EE|/

Some .idx and .sub files get different amounts of subtitle streams detected - sometimes even none

Some .idx and .sub files get different amounts of subtitle streams detected, which leads to DEL creqs, effectively removing all subtitle streams in some cases.

For some cases the issue already existed in AVD2, apparently:

Example https://anidb.net/file/884051

<?xml version="1.0" encoding="utf-8"?>
<file>
  <crc>4541710d</crc>
  <ed2k>935db5a9dc9972d87870893101435dd0</ed2k>
  <md5>8e615638ac62695af0e67a025e0c16e5</md5>
  <sha1>5f07bd6fd0c48bbb7d15ca0b3fca092edee6a678</sha1>
  <tth>igelpnqi5ag6ci2rk56aooirjakoofdeidfk4ly</tth>
  <size>16698</size>
  <avmf>
    <extension>idx</extension>
    <file_extension>idx</file_extension>
  </avmf>
</file>

Different Polynomials

Test

Multiple chapter entries for the same timestamp get reduced to one

Ver 8069

https://anidb.net/file/2099625

AVD2:

<chapters flags="None">
  <entry flags="Enabled">
    <pos start="0"/>
    <title lang="fin">BUMP OF CHICKEN: Fighter</title>
    <title lang="eng">BUMP OF CHICKEN: Fighter</title>
  </entry>
</chapters>

AVD3:

<chapters flags="None">
  <entry flags="Enabled">
    <pos start="0"/>
    <title lang="fin">BUMP OF CHICKEN: Fighter</title>
  </entry>
</chapters>

Progress bars are not updating in-place but scroll the window instead

Under certain conditions (cmd window not wide enough?) the progress bars are not update in-place but scroll the window instead.

Resulting in output like:

-------------------------------------------------------------------------------
ED2K     [#############                                                    ] 1
-------------------------------------------------------------------------------
ED2K     [##########################                                       ] 1
-------------------------------------------------------------------------------
ED2K     [#######################################                          ] 1
-------------------------------------------------------------------------------
ED2K     [####################################################             ] 1

Ctrl+C creates error dump

Running AVD3 with dotnet AVDump3CL.dll creates an error dump when aborting the process with Ctrl+C:

<AVD3CLException thrownOn="2020-05-18 21:31:22.1609">
  <Information>
    <EntryAssemblyVersion>3.0.8061.0</EntryAssemblyVersion>
    <LibVersion>3.0.8061.0</LibVersion>
    <Session>19c30e8d-a23a-4694-964f-4d737d7ddc7c</Session>
    <Framework>3.1.2</Framework>
    <OSVersion>Microsoft Windows NT 10.0.18363.0</OSVersion>
    <IntPtr.Size>8</IntPtr.Size>
    <Is64BitOperatingSystem>true</Is64BitOperatingSystem>
    <Is64BitProcess>true</Is64BitProcess>
    <ProcessorCount>4</ProcessorCount>
    <UserInteractive>true</UserInteractive>
    <SystemPageSize>4096</SystemPageSize>
    <WorkingSet>182730752</WorkingSet>
    <Commandline>E:\Programme\AVD3\AVDump3CL.dll FROMFILE arguments.txt "path"</Commandline>
  </Information>
  <Message>ConsumingStream</Message>
  <Data>
    <FileName>filename.mkv</FileName>
  </Data>
  <Cause>
    <StreamConsumerException thrownOn="2020-05-18 21:31:22.1598">
      <Message>StreamConsumer threw an Exception</Message>
      <Data>
        <StreamTag>fullpath+filename.mkv</StreamTag>
      </Data>
      <Cause>
        <AggregateException>
          <Message>One or more errors occurred. (The operation was canceled.) (The operation was canceled.) (The operation was canceled.) (The operation was canceled.) (The operation was canceled.)</Message>
          <Stacktrace />
          <Data />
          <Cause>
            <OperationCanceledException>
              <Message>The operation was canceled.</Message>
              <Stacktrace>
                <Frame>at System.Threading.CancellationToken.ThrowOperationCanceledException()</Frame>
                <Frame>at AVDump3Lib.Processing.BlockConsumers.HashCalculator.DoWork(CancellationToken ct) in /home/runner/work/AVDump3/AVDump3/AVDump3Lib/Processing/BlockConsumers/HashCalculator.cs:line 39</Frame>
                <Frame>at AVDump3Lib.Processing.BlockConsumers.BlockConsumer.ProcessBlocks(CancellationToken ct) in /home/runner/work/AVDump3/AVDump3/AVDump3Lib/Processing/BlockConsumers/BlockConsumer.cs:line 30</Frame>
              </Stacktrace>
              <Data>
                <BlockConsumerName>CRC32</BlockConsumerName>
                <BlockConsumerReadBytes>905969664</BlockConsumerReadBytes>
              </Data>
            </OperationCanceledException>
            <OperationCanceledException>
              <Message>The operation was canceled.</Message>
              <Stacktrace>
                <Frame>at System.Threading.CancellationToken.ThrowOperationCanceledException()</Frame>
                <Frame>at AVDump3Lib.Processing.BlockBuffers.BlockStream.GetBlock(Int32 consumerIndex, Int32 minBlockLength) in /home/runner/work/AVDump3/AVDump3/AVDump3Lib/Processing/BlockBuffers/BlockStream.cs:line 110</Frame>
                <Frame>at AVDump3Lib.Processing.BlockConsumers.HashCalculator.DoWork(CancellationToken ct) in /home/runner/work/AVDump3/AVDump3/AVDump3Lib/Processing/BlockConsumers/HashCalculator.cs:line 41</Frame>
                <Frame>at AVDump3Lib.Processing.BlockConsumers.BlockConsumer.ProcessBlocks(CancellationToken ct) in /home/runner/work/AVDump3/AVDump3/AVDump3Lib/Processing/BlockConsumers/BlockConsumer.cs:line 30</Frame>
              </Stacktrace>
              <Data>
                <BlockConsumerName>ED2K</BlockConsumerName>
                <BlockConsumerReadBytes>894976000</BlockConsumerReadBytes>
              </Data>
            </OperationCanceledException>
            <OperationCanceledException>
              <Message>The operation was canceled.</Message>
              <Stacktrace>
                <Frame>at System.Threading.CancellationToken.ThrowOperationCanceledException()</Frame>
                <Frame>at AVDump3Lib.Processing.BlockConsumers.HashCalculator.DoWork(CancellationToken ct) in /home/runner/work/AVDump3/AVDump3/AVDump3Lib/Processing/BlockConsumers/HashCalculator.cs:line 39</Frame>
                <Frame>at AVDump3Lib.Processing.BlockConsumers.BlockConsumer.ProcessBlocks(CancellationToken ct) in /home/runner/work/AVDump3/AVDump3/AVDump3Lib/Processing/BlockConsumers/BlockConsumer.cs:line 30</Frame>
              </Stacktrace>
              <Data>
                <BlockConsumerName>MD5</BlockConsumerName>
                <BlockConsumerReadBytes>905969664</BlockConsumerReadBytes>
              </Data>
            </OperationCanceledException>
            <OperationCanceledException>
              <Message>The operation was canceled.</Message>
              <Stacktrace>
                <Frame>at System.Threading.CancellationToken.ThrowOperationCanceledException()</Frame>
                <Frame>at AVDump3Lib.Processing.BlockConsumers.HashCalculator.DoWork(CancellationToken ct) in /home/runner/work/AVDump3/AVDump3/AVDump3Lib/Processing/BlockConsumers/HashCalculator.cs:line 39</Frame>
                <Frame>at AVDump3Lib.Processing.BlockConsumers.BlockConsumer.ProcessBlocks(CancellationToken ct) in /home/runner/work/AVDump3/AVDump3/AVDump3Lib/Processing/BlockConsumers/BlockConsumer.cs:line 30</Frame>
              </Stacktrace>
              <Data>
                <BlockConsumerName>SHA1</BlockConsumerName>
                <BlockConsumerReadBytes>905969664</BlockConsumerReadBytes>
              </Data>
            </OperationCanceledException>
            <OperationCanceledException>
              <Message>The operation was canceled.</Message>
              <Stacktrace>
                <Frame>at System.Threading.CancellationToken.ThrowOperationCanceledException()</Frame>
                <Frame>at AVDump3Lib.Processing.BlockBuffers.BlockStream.GetBlock(Int32 consumerIndex, Int32 minBlockLength) in /home/runner/work/AVDump3/AVDump3/AVDump3Lib/Processing/BlockBuffers/BlockStream.cs:line 110</Frame>
                <Frame>at AVDump3Lib.Processing.BlockConsumers.HashCalculator.DoWork(CancellationToken ct) in /home/runner/work/AVDump3/AVDump3/AVDump3Lib/Processing/BlockConsumers/HashCalculator.cs:line 41</Frame>
                <Frame>at AVDump3Lib.Processing.BlockConsumers.BlockConsumer.ProcessBlocks(CancellationToken ct) in /home/runner/work/AVDump3/AVDump3/AVDump3Lib/Processing/BlockConsumers/BlockConsumer.cs:line 30</Frame>
              </Stacktrace>
              <Data>
                <BlockConsumerName>TTH</BlockConsumerName>
                <BlockConsumerReadBytes>889192448</BlockConsumerReadBytes>
              </Data>
            </OperationCanceledException>
          </Cause>
        </AggregateException>
      </Cause>
      <Stacktrace>
        <Frame>at AVDump3Lib.Processing.StreamConsumer.StreamConsumer.ConsumeStream(IProgress`1 progress, CancellationToken ct) in /home/runner/work/AVDump3/AVDump3/AVDump3Lib/Processing/StreamConsumer/StreamConsumer.cs:line 84</Frame>
        <Frame>at AVDump3Lib.Processing.StreamConsumer.StreamConsumerCollection.ConsumeStream(ProvidedStream providedStream, IBytesReadProgress progress, CancellationToken ct) in /home/runner/work/AVDump3/AVDump3/AVDump3Lib/Processing/StreamConsumer/StreamConsumerCollection.cs:line 98</Frame>
      </Stacktrace>
    </StreamConsumerException>
  </Cause>
  <Stacktrace />
</AVD3CLException>

The path 'XYZ.AniDBReport.xml' is too long, or a component of the specified path is too long.

https://anidb.net/admin/avmf/error/?do=xml&id=723793

These errors are local errors and really not all that important for us. Is there a way to log these kinds of errors just locally but not upload them to the server?

Add new placeholders for --ReportFileName parameter

In addition to the already existing placeholders "FileName, FileNameWithoutExtension, FileExtension, ReportName, ReportFileExtension" add two more placeholders:

ED2KHash (enables ED2k consumer)
FileSize (the size of the file in bytes)

This would allow the construction of unique report filenames.

Reimplement OpenAniDBLink parameter

Please implement the parameter to open the file's AniDB link/page again.

Wrong audio date rate detected

For a file with
Duration: 01:19:47.01, start: 0.000000, bitrate: 299 kb/s
Stream #0:0: Video: msmpeg4v3 (MP43 / 0x3334504D), yuv420p, 320x240, 284 kb/s, 30.04 fps, 30.04 tbr, 30.04 tbn, 30.04 tbc
Stream #0:1: Audio: wmav2 (a[1][0][0] / 0x0161), 8000 Hz, mono, fltp, 8 kb/s
avd3 created a creq to move the audio up to 1575kbit/s (see 13395761). avd2 submitted the original and correct 8kbit/s.

`Filediscovery: Path not found` when space in path, and path ends with `\` (Windows 10)

Environment

OS: Windows 10 1904
AVD Version: 8188 (From AniDB wiki download link)
Tested in Command Prompt and Windows Terminal
Tests were performed on a SMB share, but I noted earlier that I had issues with a space in names when scanning from a local NTFS drive

Writeup

So this is rather odd, and I'm not entirely sure if the title makes sense, so I've included my test cases at the bottom of the issue. Please note that I've tried both ' and " to see if there's any difference on the failing paths^†. I've also included paths with no spaces in them, using various combinations of ', ", and trailing \, to ensure that it is actually the space causing the error.

I'd also like to note that when printing the Filediscovery error when a space is in the directory, it prints the trailing ' or ", whereas when there is no space, it removes the character and just prints the path. I don't know if this is related, or useful at all, but I've included additions test paths for a true non-existent directory (X:\Anime\NotThere\ in case it wasn't obvious -- all other folders used exist on the file system) which shows what the 'normal' output for the error shows. You can also note that the error for the directory that actually doesn't exist properly prints the trailing \, while it's missing from a directory with a space.

† It seems like using Command Prompt, it won't accept ' at all -- which could be a separate issue. I'm not sure if anything in cmd will accept paths surrounded by ', or if it just accepts ". I've tried both because I'm using Terminal, which accepts paths using either. Note the test using ' instead of " returns a " trailing the path in the error, which leads me to believe that Terminal is just converting the ' into a " before the executable receives it, but I'm not sure about that.

Sorry that this issue is kinda a massive info dump. I tried to include anything I might find useful as a developer, but if there's anything else you need/want to know or have me test, just let me know. I'll also try to take a look at the code later, to see if I can try to fix it myself.

Tests

Base Command (should it even matter): .\AVDump3CL.exe --auth={snip} --Ed2kLogPath=ed2k.txt -R {path}
Paths tested:

'X:\Anime\Steins Gate\' FAILS
"X:\Anime\Steins Gate\" FAILS
'X:\Anime\Steins Gate' WORKS
"X:\Anime\Steins Gate" WORKS
'X:\Anime\Kanon\' WORKS
"X:\Anime\Kanon\" WORKS
X:\Anime\Kanon\ WORKS
'X:\Anime\Kanon' WORKS
"X:\Anime\Kanon" WORKS
X:\Anime\Kanon WORKS

Error Output:

'X:\Anime\Steins Gate\' -- Filediscovery: Path not found: X:\Anime\Steins Gate"
"X:\Anime\Steins Gate\" -- Filediscovery: Path not found: X:\Anime\Steins Gate"
'X:\Anime\NotThere\' -- Filediscovery: Path not found: X:\Anime\NotThere\
'X:\Anime\NotThere' -- Filediscovery: Path not found: X:\Anime\NotThere
X:\Anime\NotThere\ -- Filediscovery: Path not found: X:\Anime\NotThere\
X:\Anime\NotThere -- Filediscovery: Path not found: X:\Anime\NotThere

Wrong ED2K hash for small files

Corrupt OGM file throws "TimeSpan overflowed because the duration is too long."

8152: https://anidb.net/admin/avmf/error/?do=xml&id=723950
8134: https://anidb.net/admin/avmf/error/?do=xml&id=723947

https://anidb.net/file/42782

[23:23:18] <&worf> oh wow .. that Trigun ep10 file is so broken .. it was never dumped before, with any avdump version
[23:23:37] <&worf> mind uploading that for Arokh somewhere and sending him the link, zeromind?
[23:28:29] <+zeromind> yeah, slighly corrupt
[23:29:00] <&worf> is it even playable?
[23:29:09] <+zeromind> yes
[23:29:55] <+zeromind> bunch of [ffmpeg/demuxer] ogg: CRC mismatch!

[23:34:34] <zeromind> Arokh: put broken trigun ogm e into your home dir on the avdump vm, does that work for you?

Unsupported hash consumers print no error messages

Using CRC32C on a processor that doesn’t have the requiired instructions or using CRC32-Native on Linux will make AVD3 exit silently without any indication for what’s wrong.

"Error System.InvalidOperationException: MediaInfoLib couldn't open the file" because of "weird" UTF-8 character in filename

Trying to dump e.g. "(Hi10)_Space☆Dandy_-_01_(720p)_(Commie)_(Dual_Audio)_(F042204B).mkv" will throw an "Error System.InvalidOperationException: MediaInfoLib couldn't open the file" exception.

Remove the "☆" and dumping will work.

Should MIL even be called at all on such a file? It is an .mkv file afterall and shouldn't require MIL?

AVD3 produces wrong ed2k hash(es) for files with a filesize that is a multiple of 9728000 - ed2k_alt missing from xml

v8188

https://anidb.net/admin/avmf/file/761921

AVD2: https://anidb.net/admin/avmf/1079462/?do=xml

<crc>3e0c3aac</crc>
<ed2k>7327220ef6a049a876f430a25ba498d6</ed2k>
<ed2k_alt>915c361308b30b2482148410cbb1ab99</ed2k_alt>
<md5>8b7c23164ad6684e8a6c3ed90cef9447</md5>
<sha1>5d420127418752086c77b7ab0543e05a7709c157</sha1>
<tth>5hf577krec6ykl4yeo2fq6ue2guc3r7icucxlya</tth>
<size>252928000</size>

AVD3: https://anidb.net/admin/avmf/5001822/?do=xml

<crc>3e0c3aac</crc>
<ed2k>7e03600fb071b5ef5d398b8c5d63717a</ed2k>
<md5>8b7c23164ad6684e8a6c3ed90cef9447</md5>
<sha1>5d420127418752086c77b7ab0543e05a7709c157</sha1>
<tth>5hf577krec6ykl4yeo2fq6ue2guc3r7icucxlya</tth>
<size>252928000</size>

While AVD2 calculated the red and blue variants of the ed2k hash, AVD3 calculates a completely different ed2k hash.
Also, the <ed2k_alt/> node is missing from the XML.

"MediaInfoLib couldn't open the file" - Path too long

https://anidb.net/admin/avmf/error/?do=xml&id=723965

<FileName>The Disastrous Life of Saiki K - 1x19 - Hurray! Tsundere Grandpa + Hip, Hip, Hurray! Tsundere Grandpa + Welcome to the Farthest Amusement Park! + See You Again! Saying Goodbye to the Grandparents + Congratulations on the Video Game Release! Kunio Sai.mkv</FileName>

The filename alone already exceeds the path length limit under Windows. Not surprising that MIL can't open the file.

Report locally but not upload the error to the server?

Wrong ed2k links are generated from symlinks on linux

notice 130 instead of actual file size of what symlink points too

"AniDBReport" output is missing some bits AVD2 provided

v8188/8236

Using AVD2 with the --Log parameter gave me the following output in the logfile:

F:\Anime (DDD)\test\aaabbb_\Gasshin Sentai Mechander Robo - 23 Il sacrificio [76D8265C].avi
<?xml version="1.0" encoding="utf-16"?>
<file>
  <crc>76d8265c</crc>
  <ed2k>3362c78e0a4ffcd2b52ceb164a150eb4</ed2k>
  <ed2k_alt>57fd0ca8153e8dc7d50f1c3137532165</ed2k_alt>
  <md5>ec74189188e20a3ff8bf4ca162ba0ac9</md5>
  <sha1>3384eba9b19aa94a4aeed2868c939a1beef8c5d5</sha1>
  <tth>qgyokwrdsin2ibxmsf6mmtqrx22yf2vt5o64wva</tth>
  <size>262656000</size>
  <avmf>
    [...]
  </avmf>
</file>

AVD3 with the following arguments (among others)

--Reports=AniDBReport
--ReportDirectory=Reports
--ReportFilename=report_${FileExtension}.txt

is giving me the following output in the logfile:

<file>
  <crc>76d8265c</crc>
  <ed2k>3362c78e0a4ffcd2b52ceb164a150eb4</ed2k>
  <ed2k_alt>57fd0ca8153e8dc7d50f1c3137532165</ed2k_alt>
  <md5>ec74189188e20a3ff8bf4ca162ba0ac9</md5>
  <sha1>3384eba9b19aa94a4aeed2868c939a1beef8c5d5</sha1>
  <tth>qgyokwrdsin2ibxmsf6mmtqrx22yf2vt5o64wva</tth>
  <size>262656000</size>
  <avmf>
    [...]
  </avmf>
</file>

As you can see, the following two lines, that were present in the AVD2 output, are missing in the AVD3 output:

F:\Anime (DDD)\test\aaabbb_\Gasshin Sentai Mechander Robo - 23 Il sacrificio [76D8265C].avi
<?xml version="1.0" encoding="utf-16"?>

While the <?xml> element isn't that important, it would be nice to know which file the XML in the log is associated with.

Console cursor visibility is not reset after program crash

Wrong ed2k links are generated from symlinks on linux

Continuation of #51 apparently
it wasnt really fixed, 8293 still produces incorrect ed2k links from symlinks, just in a different way

notice the file size?

Count cannot be less than zero. (Parameter 'repeatCount')

https://anidb.net/admin/avmf/error/?do=xml&id=723949
https://anidb.net/admin/avmf/error/?do=xml&id=723948
https://anidb.net/admin/avmf/error/?do=xml&id=723786

<Commandline>O:\Profile\AVDump3\AVDump3CL.dll FROMFILE O:\Profile\AVDump3\arguments.txt .</Commandline>

Possibly an issue with "." given as the path? "--PrintDiscoveredFiles" doesn't show anything weird though. https://i.imgur.com/1Uaf4nt.png (ignore the "System.NullReferenceException")

Exception when processing ogm files (OGMAudioHeader)

Files.zip

Obfuscate apikey in uploaded error reports when --IncludePersonalData is enabled

If --IncludePersonalData is enabled the user's apikey is potentially included in uploaded error reports.

Avd3 should obfuscate them since they are not required and kind of sensitive data.

A simple regexp-based replacement of the apikey should suffice.

Help text is unreadable

Help text is printed in black. On a black terminal background, it’s unreadable. (Tested on Linux)

UnauthorizedAccessExceptions

Ver 8066

Running AVD3 over an entire hdd will throw "UnauthorizedAccessExceptions" errors.

2 2020-05-21T23:28:46 "Access to the path 'M:\$RECYCLE.BIN\<user_windows_sid_here>' is denied."
2 2020-05-21T23:28:46 "Access to the path 'M:\System Volume Information' is denied."

ED2K exception when the filesize is divisible by its blocksize

--Nullstreamtest throws exception

<?xml version="1.0" encoding="utf-8"?>
<AVD3CLException thrownOn="2020-08-10 00:02:06.7744">
  <Information>
    <EntryAssemblyVersion>3.0.8134.0</EntryAssemblyVersion>
    <LibVersion>3.0.8134.0</LibVersion>
    <Session>d370b87c-54e2-40bd-b44e-591ee42cea18</Session>
    <Framework>3.1.2</Framework>
    <OSVersion>Microsoft Windows NT 10.0.18363.0</OSVersion>
    <IntPtr.Size>8</IntPtr.Size>
    <Is64BitOperatingSystem>true</Is64BitOperatingSystem>
    <Is64BitProcess>true</Is64BitProcess>
    <ProcessorCount>4</ProcessorCount>
    <UserInteractive>true</UserInteractive>
    <SystemPageSize>4096</SystemPageSize>
    <WorkingSet>1571553280</WorkingSet>
    <Commandline>E:\Programme\AVD3\AVDump3CL.dll --NullStreamTest=16:2000:16 --consumers=ed2k</Commandline>
  </Information>
  <Message>Unhandled AppDomain wide Exception</Message>
  <Data />
  <Cause>
    <AggregateException>
      <Message>One or more errors occurred. (Couldn't create Mirrored Buffer) (Couldn't create Mirrored Buffer) (Couldn't create Mirrored Buffer)</Message>
      <Stacktrace>
        <Frame>at System.AggregateException.Handle(Func`2 predicate)</Frame>
        <Frame>at AVDump3Lib.Processing.StreamConsumer.StreamConsumerCollection.ConsumeStreams(IBytesReadProgress progress, CancellationToken ct) in D:\Projects\C#\AVDump3\AVDump3Lib\Processing\StreamConsumer\StreamConsumerCollection.cs:line 0</Frame>
        <Frame>at AVDump3CL.AVD3CLModule.Process(String[] paths) in D:\Projects\C#\AVDump3\AVDump3CL\AVD3CLModule.cs:line 332</Frame>
        <Frame>at AVDump3CL.Program.Main(String[] args) in D:\Projects\C#\AVDump3\AVDump3CL\Program.cs:line 110</Frame>
      </Stacktrace>
      <Data />
      <Cause>
        <Exception>
          <Message>Couldn't create Mirrored Buffer</Message>
          <Stacktrace>
            <Frame>at AVDump3Lib.Processing.BlockBuffers.MirroredBuffer..ctor(Int32 length) in D:\Projects\C#\AVDump3\AVDump3Lib\Processing\BlockBuffers\MirroredBuffer.cs:line 41</Frame>
            <Frame>at AVDump3Lib.Processing.BlockBuffers.MirroredBufferPool.Take() in D:\Projects\C#\AVDump3\AVDump3Lib\Processing\BlockBuffers\MirroredBufferPool.cs:line 24</Frame>
            <Frame>at AVDump3Lib.Processing.StreamConsumer.StreamConsumerFactory.Create(ProvidedStream providedStream) in D:\Projects\C#\AVDump3\AVDump3Lib\Processing\StreamConsumer\StreamConsumerFactory.cs:line 34</Frame>
            <Frame>at AVDump3Lib.Processing.StreamConsumer.StreamConsumerCollection.ConsumeStream(ProvidedStream providedStream, IBytesReadProgress progress, CancellationToken ct) in D:\Projects\C#\AVDump3\AVDump3Lib\Processing\StreamConsumer\StreamConsumerCollection.cs:line 96</Frame>
            <Frame>at AVDump3Lib.Processing.StreamConsumer.StreamConsumerCollection.&lt;&gt;c__DisplayClass15_2.&lt;ConsumeStreams&gt;b__2() in D:\Projects\C#\AVDump3\AVDump3Lib\Processing\StreamConsumer\StreamConsumerCollection.cs:line 62</Frame>
            <Frame>at System.Threading.Tasks.Task.InnerInvoke()</Frame>
            <Frame>at System.Threading.Tasks.Task.&lt;&gt;c.&lt;.cctor&gt;b__274_0(Object obj)</Frame>
            <Frame>at System.Threading.ExecutionContext.RunInternal(ExecutionContext executionContext, ContextCallback callback, Object state)</Frame>
            <Frame>--- End of stack trace from previous location where exception was thrown ---</Frame>
            <Frame>at System.Threading.ExecutionContext.RunInternal(ExecutionContext executionContext, ContextCallback callback, Object state)</Frame>
            <Frame>at System.Threading.Tasks.Task.ExecuteWithThreadLocal(Task&amp; currentTaskSlot, Thread threadPoolThread)</Frame>
          </Stacktrace>
          <Data />
        </Exception>
      </Cause>
    </AggregateException>
  </Cause>
  <Stacktrace />
</AVD3CLException>

	public void WriteProgress(AVD3ConsoleProgressBuilder pb) {
	if(state == 0) {
	prevP = curP;
	curP = getProgress();
	}
	var interpolationFactor = (state + 1) / (double)UpdatePeriodInTicks;
	state++;
	state %= UpdatePeriodInTicks;

	var processedMiBsInInterval = ((curP.BytesProcessed - prevP.BytesProcessed) >> 20) / UpdatePeriodInTicks;

	if((DateTimeOffset.UtcNow - curP.StartedOn).TotalMinutes < 1) {
	var now = DateTimeOffset.UtcNow;
	var prevSpeed = (prevP.BytesProcessed >> 20) / (now - prevP.StartedOn).TotalSeconds;
	var curSpeed = (curP.BytesProcessed >> 20) / (now - curP.StartedOn).TotalSeconds;
	totalSpeedAverages[0] = (float)(prevSpeed + interpolationFactor * (curSpeed - prevSpeed));

	} else {
	var maInterval = MeanAverageMinuteInterval;
	totalSpeedAverages[0] = totalSpeedAverages[0] * (maInterval - 1) / maInterval + (processedMiBsInInterval / (float)AVD3Console.TickPeriod * 1000) / maInterval;
	}
	if((DateTimeOffset.UtcNow - curP.StartedOn).TotalMinutes < 5) {
	totalSpeedAverages[1] = totalSpeedAverages[0];
	} else {
	var maInterval = MeanAverageMinuteInterval * 5;
	totalSpeedAverages[1] = totalSpeedAverages[1] * (maInterval - 1) / maInterval + (processedMiBsInInterval / (float)AVD3Console.TickPeriod * 1000) / maInterval;
	}
	if((DateTimeOffset.UtcNow - curP.StartedOn).TotalMinutes < 15) {
	totalSpeedAverages[2] = totalSpeedAverages[1];
	} else {
	var maInterval = MeanAverageMinuteInterval * 15;
	totalSpeedAverages[2] = totalSpeedAverages[2] * (maInterval - 1) / maInterval + (processedMiBsInInterval / (float)AVD3Console.TickPeriod * 1000) / maInterval;
	}
	if(totalSpeedAverages[0] > 9999999 \|\| totalSpeedAverages[1] > 9999999 \|\| totalSpeedAverages[2] > 9999999) {
	totalSpeedDisplayAverages[0] = (int)totalSpeedAverages[0] >> 20;
	totalSpeedDisplayAverages[1] = (int)totalSpeedAverages[1] >> 20;
	totalSpeedDisplayAverages[2] = (int)totalSpeedAverages[2] >> 20;
	totalSpeedDisplayUnit = "TiB/s";

	} else if(totalSpeedAverages[0] > 9999 \|\| totalSpeedAverages[1] > 9999 \|\| totalSpeedAverages[2] > 9999) {
	totalSpeedDisplayAverages[0] = (int)totalSpeedAverages[0] >> 10;
	totalSpeedDisplayAverages[1] = (int)totalSpeedAverages[1] >> 10;
	totalSpeedDisplayAverages[2] = (int)totalSpeedAverages[2] >> 10;
	totalSpeedDisplayUnit = "GiB/s";

	} else {
	totalSpeedDisplayAverages[0] = (int)totalSpeedAverages[0];
	totalSpeedDisplayAverages[1] = (int)totalSpeedAverages[1];
	totalSpeedDisplayAverages[2] = (int)totalSpeedAverages[2];
	totalSpeedDisplayUnit = "MiB/s";
	}

	Display(pb, interpolationFactor);

	pb.SpecialJitterEvent = state == 0;
	}