Sitelet https://github.com/NVIDIA/Model-Optimizer/commit/7a2344fb9fb4b27143b7ffb0aaccd4ce81bc7313
Skip to content

Commit 7a2344f

Browse files
committed
Keep non-serve vLLM commands untouched by fakequant settings
Signed-off-by: Kinjal Patel <kinjalpravin@nvidia.com>
1 parent a096a2f commit 7a2344f

2 files changed

Lines changed: 22 additions & 5 deletions

File tree

‎examples/vllm_serve/vllm_serve_fakequant.py‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -263,12 +263,29 @@ def _find_serve_model(rest_argv: list) -> str | None:
263263
return getattr(args, "model_tag", None) or getattr(args, "model", None)
264264

265265

266+
def _run_vllm_cli(argv: list[str]) -> None:
267+
"""Delegate arguments to the stock vLLM CLI."""
268+
sys.argv = ["vllm", *argv]
269+
from vllm.entrypoints.cli.main import main as vllm_main
270+
271+
vllm_main()
272+
273+
266274
def main():
275+
argv = sys.argv[1:]
276+
# Non-serve commands do not consume ModelOpt settings, including env defaults.
277+
if argv and argv[0] in _VLLM_SUBCOMMANDS and argv[0] != "serve":
278+
_run_vllm_cli(argv)
279+
return
280+
267281
modelopt_parser = FlexibleArgumentParser(add_help=False)
268282
_add_fakequant_args(modelopt_parser)
269283
add_mlflow_args(modelopt_parser)
270284
modelopt_args, rest_argv = modelopt_parser.parse_known_args(sys.argv[1:])
271285
rest_argv = _default_to_serve(rest_argv)
286+
if rest_argv and rest_argv[0] != "serve":
287+
_run_vllm_cli(argv)
288+
return
272289

273290
if (modelopt_args.modelopt_quant_cfg or modelopt_args.modelopt_kv_quant_cfg) and (
274291
modelopt_args.modelopt_recipe_path
@@ -323,10 +340,7 @@ def main():
323340
if not _has_flag(rest_argv, "--moe-backend", "--moe_backend"):
324341
rest_argv = [*rest_argv, "--moe_backend", "triton"]
325342

326-
sys.argv = ["vllm", *rest_argv]
327-
from vllm.entrypoints.cli.main import main as vllm_main
328-
329-
vllm_main()
343+
_run_vllm_cli(rest_argv)
330344

331345

332346
if __name__ == "__main__":

‎tests/gpu_vllm/torch/quantization/test_vllm_dynamic_modules.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -210,7 +210,8 @@ def test_vllm_serve_parser_dependency_error_propagates(monkeypatch):
210210
assert raised.value is dependency_error
211211

212212

213-
def test_fakequant_launcher_passes_through_vllm_launch(monkeypatch):
213+
@pytest.mark.parametrize("quant_cfg", [None, "FP8_DEFAULT_CFG"])
214+
def test_fakequant_launcher_passes_through_vllm_launch(monkeypatch, quant_cfg):
214215
for key in (
215216
"QUANT_CFG",
216217
"KV_QUANT_CFG",
@@ -219,6 +220,8 @@ def test_fakequant_launcher_passes_through_vllm_launch(monkeypatch):
219220
"QUANT_FILE_PATH",
220221
):
221222
monkeypatch.delenv(key, raising=False)
223+
if quant_cfg:
224+
monkeypatch.setenv("QUANT_CFG", quant_cfg)
222225
launcher = _load_fakequant_launcher(monkeypatch)
223226
monkeypatch.setattr(launcher, "resolve_mlflow_args", Mock())
224227
vllm_main = Mock()

0 commit comments

Comments
 (0)