decodes outputs using tokenizer

app.py

@@ -113,16 +113,40 @@ class LOperatorDemo:
                 tokenize=True,
             )
             
-            logger.info(f"Processor output type: {type(inputs)}")
+            logger.info(f"Processor output keys: {list(inputs.keys())}")
             
-            # If processor returns a string, just return it directly
-            if isinstance(inputs, str):
-                logger.info("Processor returned string, returning directly...")
-                return inputs
+            # Move inputs to device
+            for key, value in inputs.items():
+                if isinstance(value, torch.Tensor):
+                    inputs[key] = value.to(self.model.device)
             
-            # If it's a dict or other type, convert to string and return
-            logger.info("Converting processor output to string...")
-            return str(inputs)
+            logger.info(f"Inputs shape: {inputs['input_ids'].shape}, device: {inputs['input_ids'].device}")
+            
+            # Generate response
+            logger.info("Generating response...")
+            with torch.no_grad():
+                outputs = self.model.generate(
+                    **inputs,
+                    max_new_tokens=128,
+                    do_sample=True,
+                    temperature=0.7,
+                    top_p=0.9,
+                    pad_token_id=self.processor.tokenizer.eos_token_id
+                )
+            
+            logger.info("Decoding response...")
+            # Decode the generated tokens
+            response = self.processor.tokenizer.decode(
+                outputs[0][inputs['input_ids'].shape[1]:], 
+                skip_special_tokens=True
+            )
+            
+            # Try to parse as JSON for better formatting
+            try:
+                parsed_response = json.loads(response)
+                return json.dumps(parsed_response, indent=2)
+            except:
+                return response
                 
         except Exception as e:
             logger.error(f"Error generating action: {str(e)}")