diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 4905674ad..33df7ef18 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -221,7 +221,7 @@ def _real_main(argv=None):
             'key': 'MetadataFromTitle',
             'titleformat': opts.metafromtitle
         })
-    if opts.addmetadata:
+    if opts.addmetadata and opts.keepvideo:
         postprocessors.append({'key': 'FFmpegMetadata'})
     if opts.extractaudio:
         postprocessors.append({
@@ -230,6 +230,8 @@ def _real_main(argv=None):
             'preferredquality': opts.audioquality,
             'nopostoverwrites': opts.nopostoverwrites,
         })
+        if opts.addmetadata:
+            postprocessors.append({'key': 'FFmpegMetadata'})
     if opts.recodevideo:
         postprocessors.append({
             'key': 'FFmpegVideoConvertor',