diff --git a/src/Service/Transifex/TranslationsService.php b/src/Service/Transifex/TranslationsService.php
index ad46c7f..085ad9c 100644
--- a/src/Service/Transifex/TranslationsService.php
+++ b/src/Service/Transifex/TranslationsService.php
@@ -52,6 +52,7 @@ public function getTranslations(TranslationDTO $translationDTO, LoggerInterface
 
                 sleep(2 ** $attempt);
             } catch (InvalidFileException|RegexException $e) {
+                $this->filesystem->remove($filePath);
                 $this->outputErrors($logger, $filePath, $e->getMessage());
                 break;
             }
diff --git a/tests/Functional/Command/MauticLanguagePackerCommandTest.php b/tests/Functional/Command/MauticLanguagePackerCommandTest.php
index 6d1c8a4..5a43b94 100644
--- a/tests/Functional/Command/MauticLanguagePackerCommandTest.php
+++ b/tests/Functional/Command/MauticLanguagePackerCommandTest.php
@@ -430,7 +430,10 @@ public static function provideExecutionData(): iterable
         ];
     }
 
-    public function testPackageZipFolderStructure(): void
+    /**
+     * @dataProvider providePackageZipData
+     */
+    public function testPackageZipFolderStructure(string $iniBody, bool $removeIniFile = false): void
     {
         $container       = self::getContainer();
         $transifexConfig = $container->get(ConfigInterface::class);
@@ -468,7 +471,7 @@ public function testPackageZipFolderStructure(): void
                 array_merge(['Content-Length' => ['498']], self::getCommonHeaders())
             ),
             self::getMockResponse(
-                self::buildIniBody(),
+                self::buildIniBody($iniBody),
                 Request::METHOD_GET,
                 "https://rest.api.transifex.com/resource_translations_async_downloads/$uuid",
                 self::getCommonHeaders()
@@ -497,6 +500,12 @@ public function testPackageZipFolderStructure(): void
             'af/config.json',
             'af/config.php',
         ];
+
+        if ($removeIniFile) {
+            unset($expectedFolderStructure[1]);
+            $expectedFolderStructure = array_values($expectedFolderStructure);
+        }
+
         $cnt = 0;
 
         foreach ($packagesDirFinder as $file) {
@@ -515,6 +524,17 @@ public function testPackageZipFolderStructure(): void
         Assert::assertSame(count($expectedFolderStructure), $cnt);
     }
 
+    /**
+     * @return array<mixed>
+     */
+    public static function providePackageZipData(): iterable
+    {
+        yield ['mautic.addon.notice.reloaded="%added% addons were added, %updated% updated, and %disabled% disabled."', false];
+
+        // invalid ini should get skip in the zip
+        yield ['mautic.addon.notice.reloaded="%added% addons were added, %updated% updated, and %disabled% disabled.', true];
+    }
+
     /**
      * @return array<string, array<string>>
      */