Deploy Quantized Models to Factory Edge Devices with vLLM and ExecuTorch

                      
                     
import os
import torch
import vllm
from execurotch import ExecuTorch

# Configuration
MODEL_PATH = os.getenv('MODEL_PATH', 'model.pt')  # Path to the quantized model
DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'  # Use GPU if available

# Initialize vLLM and ExecuTorch
vllm_model = vllm.Model(model_path=MODEL_PATH).to(DEVICE)
execu_torch = ExecuTorch(model=vllm_model)

# Function to process input and make predictions
def predict(input_data: str) -> str:
    try:
        # Preprocess input data
        inputs = vllm_model.tokenize(input_data)
        # Make prediction
        with torch.no_grad():
            output = execu_torch(inputs)
        # Process the output
        return vllm_model.decode(output)
    except Exception as e:
        print(f'Error during prediction: {e}')  # Log the error
        return 'Prediction failed'

# Main execution
if __name__ == '__main__':
    # Example input
    sample_input = 'This is a sample input.'
    result = predict(sample_input)
    print(f'Prediction result: {result}')